Audio cleanup job created

client/src/app/app.module.ts

@@ -48,6 +48,7 @@ import { ResetComponent } from './components/reset/reset.component';
 import { ProfileComponent } from './components/profile/profile.component';
 import { AboutComponent } from './components/about/about.component';
 import { FooterComponent } from './components/footer/footer.component';
+import { JobsService } from 'app/services/jobs.service';
 
 export function authHttpServiceFactory(http: Http, options: RequestOptions) {
     return new AuthHttp(
@@ -122,6 +123,7 @@ export function authHttpServiceFactory(http: Http, options: RequestOptions) {
         PodcastService,
         ImageService,
         DebugService,
+        JobsService,
         GlobalsService
     ],
     bootstrap: [AppComponent]

client/src/app/components/debug/debug.component.html

@@ -20,6 +20,17 @@
                     </div>
                 </div>
             </div>
+            <div class="row">
+                <div class="block">
+                    <div class="block-header">
+                        Jobs
+                    </div>
+                    <div class="block-content">
+                        <button class="btn btn-primary"
+                                (click)="processOrphans()">Process Orphans</button>
+                    </div>
+                </div>
+            </div>
         </div>
         <div class="col-md-6">
             <div class="block">
@@ -30,11 +41,11 @@
                     </pre>
                     </div>
                     <div class="row">
-                        API Host: {{apiHost}} <br /> 
+                        API Host: {{apiHost}} <br />
-                        SignalR Host: {{signalrHost}} <br /> 
+                        SignalR Host: {{signalrHost}} <br />
                         Ping: {{pingPong}} </div>
                 </div>
             </div>
         </div>
     </div>
-</div>
+</div>

client/src/app/components/debug/debug.component.ts

@@ -3,6 +3,7 @@ import { SignalRService } from 'app/services/signalr.service';
 import { Component, OnInit } from '@angular/core';
 import { DebugService } from 'app/services/debug.service';
 import { environment } from 'environments/environment';
+import { JobsService } from 'app/services/jobs.service';
 
 @Component({
     selector: 'app-debug',
@@ -18,7 +19,8 @@ export class DebugComponent implements OnInit {
     signalrHost = environment.SIGNALR_HOST;
     pingPong = '';
 
-    constructor(private _debugService: DebugService, private _signalrService: SignalRService) {}
+    constructor(private _debugService: DebugService, private _jobsService: JobsService,
+        private _signalrService: SignalRService) {}
     ngOnInit() {
         this._signalrService
             .init(`${environment.SIGNALR_HOST}hubs/debug`)
@@ -41,4 +43,9 @@ export class DebugComponent implements OnInit {
     doSomething() {
         alert('doSomething was did');
     }
+
+    processOrphans(){
+        this._jobsService.processOrphans()
+            .subscribe(e => console.log('debug.component.ts', 'processOrphans', e));
+    }
 }

client/src/app/components/navbar/navbar.component.html

@@ -4,6 +4,9 @@
             <a [routerLink]="['add']">
                 <i class="fa fa-plus"></i> Add Podcast
             </a>
+            <!-- <a [routerLink]="['debug']">
+                <i class="fa fa-plus"></i> Debug
+            </a> -->
         </div>
         <div class="content-header-section" *ngIf="user$ | async; let user; else loading">
             <img [src]="user.profileImage"
@@ -25,4 +28,4 @@
         </div>
         <ng-template #loading><i class="fa fa-sun-o fa-spin text-white"></i></ng-template>
     </div>
-</header>
+</header>

client/src/app/components/podcast/podcast-add-url-form/podcast-add-url-form.component.ts

@@ -17,6 +17,7 @@ import { PodcastService } from 'app/services/podcast.service';
 export class PodcastAddUrlFormComponent implements AfterViewInit {
     @Input() podcast: PodcastModel;
     @Output() onUrlAddComplete: EventEmitter<any> = new EventEmitter();
+    @Output() onUploadDeferred: EventEmitter<any> = new EventEmitter();
     newEntrySourceUrl: string;
     errorText: string;
     isPosting: boolean = false;
@@ -36,20 +37,24 @@ export class PodcastAddUrlFormComponent implements AfterViewInit {
         this.errorText = '';
         if (this.isValidURL(urlToCheck)) {
             this.isPosting = true;
-            const entry = new PodcastEntryModel(
+            const entry = new PodcastEntryModel(this.podcast.id, urlToCheck);
-                this.podcast.id,
+            this._service.addEntry(entry).subscribe(
-                urlToCheck
+                e => {
-            );
+                    debugger;
-            this._service.addEntry(entry)
-                .subscribe(e => {
                     if (e) {
-                        this.onUrlAddComplete.emit(e);
+                        if (e.ProcessingStatus == 6) {
+                            this.onUploadDeferred.emit(e);
+                        } else {
+                            this.onUrlAddComplete.emit(e);
+                        }
                     }
-                }, (err) => {
+                },
+                err => {
                     this.isPosting = false;
                     this.errorText = 'This does not look like a valid URL';
                     this.newEntrySourceUrl = urlToCheck;
-                });
+                }
+            );
         } else {
             this.isPosting = false;
             this.errorText = 'This does not look like a valid URL';

client/src/app/components/podcast/podcast.component.html

@@ -45,7 +45,8 @@
             </app-podcast-upload-form>
             <app-podcast-add-url-form *ngIf="urlMode"
                 [podcast]="podcast"
-                (onUrlAddComplete)="onUrlAddComplete($event)">
+                (onUrlAddComplete)="onUrlAddComplete($event)"
+                (onUploadDeferred)="onEntryUploadDeferred($event)">
             </app-podcast-add-url-form>
             <table class="js-table-checkable table table-hover js-table-checkable-enabled">
                 <tbody>

client/src/app/components/podcast/podcast.component.ts

@@ -7,7 +7,7 @@ import { AppComponent } from 'app/app.component';
 import { Store } from '@ngrx/store';
 import { ApplicationState } from 'app/store';
 import { HostListener } from '@angular/core';
-import {Location} from '@angular/common';
+import { Location } from '@angular/common';
 
 import { UpdateAction, AddAction } from 'app/actions/entries.actions';
 import * as fromPodcast from 'app/reducers';
@@ -34,7 +34,11 @@ export class PodcastComponent {
         }
     }
 
-    constructor(private _store: Store<ApplicationState>, route: ActivatedRoute, private _location: Location) {
+    constructor(
+        private _store: Store<ApplicationState>,
+        route: ActivatedRoute,
+        private _location: Location
+    ) {
         this.selectedPodcast$ = _store.select(fromPodcast.getSelectedPodcast);
 
         this.entries$ = _store.select(fromPodcast.getEntries);
@@ -48,8 +52,12 @@ export class PodcastComponent {
                 this.selectedPodcast$.subscribe(r => {
                     if (r) {
                         slug = r.slug;
-                        _store.dispatch(new fromEntriesActions.LoadAction(slug));
+                        _store.dispatch(
-                        _store.dispatch(new fromPodcastActions.SelectAction(slug));
+                            new fromEntriesActions.LoadAction(slug)
+                        );
+                        _store.dispatch(
+                            new fromPodcastActions.SelectAction(slug)
+                        );
 
                         this._location.go('/podcasts/' + slug);
                     }
@@ -79,6 +87,9 @@ export class PodcastComponent {
         // so do a funky success/update dance
         this._store.dispatch(new fromEntriesActions.AddSuccessAction(entry));
         this._store.dispatch(new fromEntriesActions.UpdateAction(entry));
+    }
+    onEntryUploadDeferred($event) {
+
     }
     onUrlAddComplete(entry: PodcastEntryModel) {
         this.urlMode = false;

server/Controllers/EntryController.cs

@@ -11,6 +11,7 @@ using Microsoft.Extensions.Options;
 using PodNoms.Api.Models;
 using PodNoms.Api.Models.ViewModels;
 using PodNoms.Api.Persistence;
+using PodNoms.Api.Services;
 using PodNoms.Api.Services.Processor;
 using PodNoms.Api.Services.Storage;
 
@@ -67,18 +68,23 @@ namespace PodNoms.Api.Controllers {
             // first check url is valid
             var entry = _mapper.Map<PodcastEntryViewModel, PodcastEntry>(item);
             var podcast = await _podcastRepository.GetAsync(item.PodcastId);
-            if (podcast != null && await _processor.GetInformation(entry)) {
+            if (podcast != null) {
-                if (entry.ProcessingStatus == ProcessingStatus.Processing) {
+                var status = await _processor.GetInformation(entry);
-                    if (string.IsNullOrEmpty(entry.ImageUrl)) {
+                if (status == AudioType.Valid) {
-                        entry.ImageUrl = $"{_storageSettings.CdnUrl}static/images/default-entry.png";
+                    if (entry.ProcessingStatus == ProcessingStatus.Processing) {
+                        if (string.IsNullOrEmpty(entry.ImageUrl)) {
+                            entry.ImageUrl = $"{_storageSettings.CdnUrl}static/images/default-entry.png";
+                        }
+                        entry.Podcast = podcast;
+                        entry.Processed = false;
+                        await _repository.AddOrUpdateAsync(entry);
+                        await _unitOfWork.CompleteAsync();
+                        _processEntry(entry);
+                        var result = _mapper.Map<PodcastEntry, PodcastEntryViewModel>(entry);
+                        return result;
                     }
-                    entry.Podcast = podcast;
+                } else if (status == AudioType.Playlist) {
-                    entry.Processed = false;
+                    return Accepted(entry);
-                    await _repository.AddOrUpdateAsync(entry);
-                    await _unitOfWork.CompleteAsync();
-                    _processEntry(entry);
-                    var result = _mapper.Map<PodcastEntry, PodcastEntryViewModel>(entry);
-                    return result;
                 }
             }
             return BadRequest();

server/Models/PodcastEntry.cs

@@ -7,7 +7,8 @@ namespace PodNoms.Api.Models {
         Processing, //2
         Uploading, //3
         Processed, //4
-        Failed //5
+        Failed, //5
+        Deferred //6
     }
     public class PodcastEntry : BaseModel {
 

server/Services/Downloader/AudioDownloader.cs

@@ -13,6 +13,7 @@ using static NYoutubeDL.Helpers.Enums;
 
 namespace PodNoms.Api.Services.Downloader {
     public class AudioDownloader {
+
         private readonly string _url;
         private readonly string _downloader;
         public VideoDownloadInfo Properties { get; private set; }
@@ -49,17 +50,22 @@ namespace PodNoms.Api.Services.Downloader {
                 return $"{{\"Error\": \"{ex.Message}\"}}";
             }
         }
-        public async Task<bool> GetInfo() {
+        public async Task<AudioType> GetInfo() {
-            var ret = false;
+            var ret = AudioType.Invalid;
             await Task.Run(() => {
                 var youtubeDl = new YoutubeDL();
                 youtubeDl.VideoUrl = this._url;
                 DownloadInfo info = youtubeDl.GetDownloadInfo();
-                ret = (
+
-                   info != null &&
+                if (info != null &&
-                   info is VideoDownloadInfo && //make sure it's not a playlist                                                                                                 
+                   (info.Errors.Count == 0 || info.VideoSize != null)) {
-                   (info.Errors.Count == 0 || info.VideoSize != null));
+                    if (info is PlaylistDownloadInfo) {
-                if (ret) this.Properties = (VideoDownloadInfo)info;
+                        ret = AudioType.Playlist;
+                    } else if (info is VideoDownloadInfo) {
+                        ret = AudioType.Valid;
+                        this.Properties = (VideoDownloadInfo)info;
+                    }
+                }
             });
             return ret;
         }

server/Services/Jobs/ClearOrphanAudioJob.cs

@@ -7,18 +7,15 @@ using Microsoft.WindowsAzure.Storage.Blob;
 using PodNoms.Api.Models;
 using PodNoms.Api.Persistence;
 
-namespace PodNoms.Api.Services.Jobs
+namespace PodNoms.Api.Services.Jobs {
-{
+    public class ClearOrphanAudioJob : IJob {
-    public class ClearOrphanAudioJob : IJob
-    {
         public readonly IEntryRepository _entryRepository;
         public readonly StorageSettings _storageSettings;
         public readonly AudioFileStorageSettings _audioStorageSettings;
         private readonly ILogger<ClearOrphanAudioJob> _logger;
 
         public ClearOrphanAudioJob(IEntryRepository entryRepository, IOptions<StorageSettings> storageSettings,
-            IOptions<AudioFileStorageSettings> audioStorageSettings, ILoggerFactory logger)
+            IOptions<AudioFileStorageSettings> audioStorageSettings, ILoggerFactory logger) {
-        {
             this._storageSettings = storageSettings.Value;
             this._audioStorageSettings = audioStorageSettings.Value;
             this._entryRepository = entryRepository;
@@ -26,38 +23,28 @@ namespace PodNoms.Api.Services.Jobs
             this._logger = logger.CreateLogger<ClearOrphanAudioJob>();
         }
 
-        public async Task Execute()
+        public async Task Execute() {
-        {
+            try {
-            try
-            {
                 CloudStorageAccount storageAccount = CloudStorageAccount.Parse(_storageSettings.ConnectionString);
                 CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
                 CloudBlobContainer container = blobClient.GetContainerReference(_audioStorageSettings.ContainerName);
 
                 var blobs = await container.ListBlobsSegmentedAsync(null);
-                foreach (CloudBlockBlob blob in blobs.Results)
+                foreach (CloudBlockBlob blob in blobs.Results) {
-                {
+                    try {
-                    try
-                    {
                         Console.WriteLine(blob.StorageUri);
                         var guid = blob.Name.Split('.')[0];
-                        if (!string.IsNullOrEmpty(guid))
+                        if (!string.IsNullOrEmpty(guid)) {
-                        {
                             var entry = await _entryRepository.GetByUidAsync(guid);
-                            if (entry == null)
+                            if (entry == null) {
-                            {
                                 await blob.DeleteIfExistsAsync();
                             }
                         }
-                    }
+                    } catch (Exception e) {
-                    catch (Exception e)
-                    {
                         _logger.LogWarning($"Error processing blob {blob.Uri}\n{e.Message}");
                     }
                 }
-            }
+            } catch (Exception ex) {
-            catch (Exception ex)
-            {
                 _logger.LogError($"Error clearing orphans\n{ex.Message}");
             }
         }

server/Services/Processor/IUrlProcessService.cs

@@ -3,8 +3,9 @@ using PodNoms.Api.Models;
 
 namespace PodNoms.Api.Services.Processor {
     public interface IUrlProcessService {
-        Task<bool> GetInformation (int entryId);
+
-        Task<bool> GetInformation (PodcastEntry entry);
+        Task<AudioType> GetInformation(int entryId);
-        Task<bool> DownloadAudio (int entryId);
+        Task<AudioType> GetInformation(PodcastEntry entry);
+        Task<bool> DownloadAudio(int entryId);
     }
 }

server/Services/Processor/UrlProcessService.cs

@@ -37,23 +37,23 @@ namespace PodNoms.Api.Services.Processor {
                 uid,
                 e);
         }
-        public async Task<bool> GetInformation(int entryId) {
+        public async Task<AudioType> GetInformation(int entryId) {
             var entry = await _repository.GetAsync(entryId);
             if (entry == null || string.IsNullOrEmpty(entry.SourceUrl)) {
                 _logger.LogError("Unable to process item");
-                return false;
+                return AudioType.Invalid;
             }
             if (entry.SourceUrl.EndsWith(".mp3") || entry.SourceUrl.EndsWith(".wav") || entry.SourceUrl.EndsWith(".aif")) {
-                return true;
+                return AudioType.Valid;
             }
             return await GetInformation(entry);
         }
 
-        public async Task<bool> GetInformation(PodcastEntry entry) {
+        public async Task<AudioType> GetInformation(PodcastEntry entry) {
 
             var downloader = new AudioDownloader(entry.SourceUrl, _applicationsSettings.Downloader);
-            await downloader.GetInfo();
+            var ret = await downloader.GetInfo();
-            if (downloader.Properties != null) {
+            if (ret == AudioType.Valid) {
                 entry.Title = downloader.Properties?.Title;
                 entry.Description = downloader.Properties?.Description;
                 entry.ImageUrl = downloader.Properties?.Thumbnail;
@@ -68,11 +68,8 @@ namespace PodNoms.Api.Services.Processor {
 
                 _logger.LogDebug("***DOWNLOAD INFO RETRIEVED****\n");
                 _logger.LogDebug($"Title: {entry.Title}\nDescription: {entry.Description}\nAuthor: {entry.Author}\n");
-
-                // var pusherResult = await _sendProcessCompleteMessage(entry);
-                return true;
             }
-            return false;
+            return ret;
         }
         public async Task<bool> DownloadAudio(int entryId) {
             var entry = await _repository.GetAsync(entryId);

server/Services/RemoteAudioTypes.cs

@@ -0,0 +1,7 @@
+namespace PodNoms.Api.Services {
+    public enum AudioType {
+        Invalid,
+        Valid,
+        Playlist
+    }
+}

server/publish.sh

@@ -3,3 +3,4 @@ unset DOCKER_HOST
 unset DOCKER_TLS_VERIFY
 
 docker build --rm -f Dockerfile -t fergalmoran/podnoms.api . && docker push fergalmoran/podnoms.api
+docker push fergalmoran/podnoms.api