diff --git a/client/src/app/app.module.ts b/client/src/app/app.module.ts index 8d9fa91..3448dfb 100644 --- a/client/src/app/app.module.ts +++ b/client/src/app/app.module.ts @@ -48,6 +48,7 @@ import { ResetComponent } from './components/reset/reset.component'; import { ProfileComponent } from './components/profile/profile.component'; import { AboutComponent } from './components/about/about.component'; import { FooterComponent } from './components/footer/footer.component'; +import { JobsService } from 'app/services/jobs.service'; export function authHttpServiceFactory(http: Http, options: RequestOptions) { return new AuthHttp( @@ -122,6 +123,7 @@ export function authHttpServiceFactory(http: Http, options: RequestOptions) { PodcastService, ImageService, DebugService, + JobsService, GlobalsService ], bootstrap: [AppComponent] diff --git a/client/src/app/components/debug/debug.component.html b/client/src/app/components/debug/debug.component.html index 5385721..b175847 100644 --- a/client/src/app/components/debug/debug.component.html +++ b/client/src/app/components/debug/debug.component.html @@ -20,6 +20,17 @@ +
+
+
+ Jobs +
+
+ +
+
+
@@ -30,11 +41,11 @@
- API Host: {{apiHost}}
- SignalR Host: {{signalrHost}}
+ API Host: {{apiHost}}
+ SignalR Host: {{signalrHost}}
Ping: {{pingPong}}
- \ No newline at end of file + diff --git a/client/src/app/components/debug/debug.component.ts b/client/src/app/components/debug/debug.component.ts index d51a37d..16eab78 100644 --- a/client/src/app/components/debug/debug.component.ts +++ b/client/src/app/components/debug/debug.component.ts @@ -3,6 +3,7 @@ import { SignalRService } from 'app/services/signalr.service'; import { Component, OnInit } from '@angular/core'; import { DebugService } from 'app/services/debug.service'; import { environment } from 'environments/environment'; +import { JobsService } from 'app/services/jobs.service'; @Component({ selector: 'app-debug', @@ -18,7 +19,8 @@ export class DebugComponent implements OnInit { signalrHost = environment.SIGNALR_HOST; pingPong = ''; - constructor(private _debugService: DebugService, private _signalrService: SignalRService) {} + constructor(private _debugService: DebugService, private _jobsService: JobsService, + private _signalrService: SignalRService) {} ngOnInit() { this._signalrService .init(`${environment.SIGNALR_HOST}hubs/debug`) @@ -41,4 +43,9 @@ export class DebugComponent implements OnInit { doSomething() { alert('doSomething was did'); } + + processOrphans(){ + this._jobsService.processOrphans() + .subscribe(e => console.log('debug.component.ts', 'processOrphans', e)); + } } diff --git a/client/src/app/components/navbar/navbar.component.html b/client/src/app/components/navbar/navbar.component.html index 69922c9..81014ec 100644 --- a/client/src/app/components/navbar/navbar.component.html +++ b/client/src/app/components/navbar/navbar.component.html @@ -4,6 +4,9 @@ Add Podcast +
- \ No newline at end of file + diff --git a/client/src/app/components/podcast/podcast-add-url-form/podcast-add-url-form.component.ts b/client/src/app/components/podcast/podcast-add-url-form/podcast-add-url-form.component.ts index 7e110f2..97e4122 100644 --- a/client/src/app/components/podcast/podcast-add-url-form/podcast-add-url-form.component.ts +++ b/client/src/app/components/podcast/podcast-add-url-form/podcast-add-url-form.component.ts @@ -17,6 +17,7 @@ import { PodcastService } from 'app/services/podcast.service'; export class PodcastAddUrlFormComponent implements AfterViewInit { @Input() podcast: PodcastModel; @Output() onUrlAddComplete: EventEmitter = new EventEmitter(); + @Output() onUploadDeferred: EventEmitter = new EventEmitter(); newEntrySourceUrl: string; errorText: string; isPosting: boolean = false; @@ -36,20 +37,24 @@ export class PodcastAddUrlFormComponent implements AfterViewInit { this.errorText = ''; if (this.isValidURL(urlToCheck)) { this.isPosting = true; - const entry = new PodcastEntryModel( - this.podcast.id, - urlToCheck - ); - this._service.addEntry(entry) - .subscribe(e => { + const entry = new PodcastEntryModel(this.podcast.id, urlToCheck); + this._service.addEntry(entry).subscribe( + e => { + debugger; if (e) { - this.onUrlAddComplete.emit(e); + if (e.ProcessingStatus == 6) { + this.onUploadDeferred.emit(e); + } else { + this.onUrlAddComplete.emit(e); + } } - }, (err) => { + }, + err => { this.isPosting = false; this.errorText = 'This does not look like a valid URL'; this.newEntrySourceUrl = urlToCheck; - }); + } + ); } else { this.isPosting = false; this.errorText = 'This does not look like a valid URL'; diff --git a/client/src/app/components/podcast/podcast.component.html b/client/src/app/components/podcast/podcast.component.html index bd0bb75..7ffbc20 100644 --- a/client/src/app/components/podcast/podcast.component.html +++ b/client/src/app/components/podcast/podcast.component.html @@ -45,7 +45,8 @@ + (onUrlAddComplete)="onUrlAddComplete($event)" + (onUploadDeferred)="onEntryUploadDeferred($event)"> diff --git a/client/src/app/components/podcast/podcast.component.ts b/client/src/app/components/podcast/podcast.component.ts index e042ad1..5882e4d 100644 --- a/client/src/app/components/podcast/podcast.component.ts +++ b/client/src/app/components/podcast/podcast.component.ts @@ -7,7 +7,7 @@ import { AppComponent } from 'app/app.component'; import { Store } from '@ngrx/store'; import { ApplicationState } from 'app/store'; import { HostListener } from '@angular/core'; -import {Location} from '@angular/common'; +import { Location } from '@angular/common'; import { UpdateAction, AddAction } from 'app/actions/entries.actions'; import * as fromPodcast from 'app/reducers'; @@ -34,7 +34,11 @@ export class PodcastComponent { } } - constructor(private _store: Store, route: ActivatedRoute, private _location: Location) { + constructor( + private _store: Store, + route: ActivatedRoute, + private _location: Location + ) { this.selectedPodcast$ = _store.select(fromPodcast.getSelectedPodcast); this.entries$ = _store.select(fromPodcast.getEntries); @@ -48,8 +52,12 @@ export class PodcastComponent { this.selectedPodcast$.subscribe(r => { if (r) { slug = r.slug; - _store.dispatch(new fromEntriesActions.LoadAction(slug)); - _store.dispatch(new fromPodcastActions.SelectAction(slug)); + _store.dispatch( + new fromEntriesActions.LoadAction(slug) + ); + _store.dispatch( + new fromPodcastActions.SelectAction(slug) + ); this._location.go('/podcasts/' + slug); } @@ -79,6 +87,9 @@ export class PodcastComponent { // so do a funky success/update dance this._store.dispatch(new fromEntriesActions.AddSuccessAction(entry)); this._store.dispatch(new fromEntriesActions.UpdateAction(entry)); + } + onEntryUploadDeferred($event) { + } onUrlAddComplete(entry: PodcastEntryModel) { this.urlMode = false; diff --git a/server/Controllers/EntryController.cs b/server/Controllers/EntryController.cs index f0d953e..0076e2b 100644 --- a/server/Controllers/EntryController.cs +++ b/server/Controllers/EntryController.cs @@ -11,6 +11,7 @@ using Microsoft.Extensions.Options; using PodNoms.Api.Models; using PodNoms.Api.Models.ViewModels; using PodNoms.Api.Persistence; +using PodNoms.Api.Services; using PodNoms.Api.Services.Processor; using PodNoms.Api.Services.Storage; @@ -67,18 +68,23 @@ namespace PodNoms.Api.Controllers { // first check url is valid var entry = _mapper.Map(item); var podcast = await _podcastRepository.GetAsync(item.PodcastId); - if (podcast != null && await _processor.GetInformation(entry)) { - if (entry.ProcessingStatus == ProcessingStatus.Processing) { - if (string.IsNullOrEmpty(entry.ImageUrl)) { - entry.ImageUrl = $"{_storageSettings.CdnUrl}static/images/default-entry.png"; + if (podcast != null) { + var status = await _processor.GetInformation(entry); + if (status == AudioType.Valid) { + if (entry.ProcessingStatus == ProcessingStatus.Processing) { + if (string.IsNullOrEmpty(entry.ImageUrl)) { + entry.ImageUrl = $"{_storageSettings.CdnUrl}static/images/default-entry.png"; + } + entry.Podcast = podcast; + entry.Processed = false; + await _repository.AddOrUpdateAsync(entry); + await _unitOfWork.CompleteAsync(); + _processEntry(entry); + var result = _mapper.Map(entry); + return result; } - entry.Podcast = podcast; - entry.Processed = false; - await _repository.AddOrUpdateAsync(entry); - await _unitOfWork.CompleteAsync(); - _processEntry(entry); - var result = _mapper.Map(entry); - return result; + } else if (status == AudioType.Playlist) { + return Accepted(entry); } } return BadRequest(); diff --git a/server/Models/PodcastEntry.cs b/server/Models/PodcastEntry.cs index 69c9b79..e992a50 100755 --- a/server/Models/PodcastEntry.cs +++ b/server/Models/PodcastEntry.cs @@ -7,7 +7,8 @@ namespace PodNoms.Api.Models { Processing, //2 Uploading, //3 Processed, //4 - Failed //5 + Failed, //5 + Deferred //6 } public class PodcastEntry : BaseModel { diff --git a/server/Services/Downloader/AudioDownloader.cs b/server/Services/Downloader/AudioDownloader.cs index 6f0dada..8a99654 100644 --- a/server/Services/Downloader/AudioDownloader.cs +++ b/server/Services/Downloader/AudioDownloader.cs @@ -13,6 +13,7 @@ using static NYoutubeDL.Helpers.Enums; namespace PodNoms.Api.Services.Downloader { public class AudioDownloader { + private readonly string _url; private readonly string _downloader; public VideoDownloadInfo Properties { get; private set; } @@ -49,17 +50,22 @@ namespace PodNoms.Api.Services.Downloader { return $"{{\"Error\": \"{ex.Message}\"}}"; } } - public async Task GetInfo() { - var ret = false; + public async Task GetInfo() { + var ret = AudioType.Invalid; await Task.Run(() => { var youtubeDl = new YoutubeDL(); youtubeDl.VideoUrl = this._url; DownloadInfo info = youtubeDl.GetDownloadInfo(); - ret = ( - info != null && - info is VideoDownloadInfo && //make sure it's not a playlist - (info.Errors.Count == 0 || info.VideoSize != null)); - if (ret) this.Properties = (VideoDownloadInfo)info; + + if (info != null && + (info.Errors.Count == 0 || info.VideoSize != null)) { + if (info is PlaylistDownloadInfo) { + ret = AudioType.Playlist; + } else if (info is VideoDownloadInfo) { + ret = AudioType.Valid; + this.Properties = (VideoDownloadInfo)info; + } + } }); return ret; } diff --git a/server/Services/Jobs/ClearOrphanAudioJob.cs b/server/Services/Jobs/ClearOrphanAudioJob.cs index effff73..4bdddfc 100644 --- a/server/Services/Jobs/ClearOrphanAudioJob.cs +++ b/server/Services/Jobs/ClearOrphanAudioJob.cs @@ -7,18 +7,15 @@ using Microsoft.WindowsAzure.Storage.Blob; using PodNoms.Api.Models; using PodNoms.Api.Persistence; -namespace PodNoms.Api.Services.Jobs -{ - public class ClearOrphanAudioJob : IJob - { +namespace PodNoms.Api.Services.Jobs { + public class ClearOrphanAudioJob : IJob { public readonly IEntryRepository _entryRepository; public readonly StorageSettings _storageSettings; public readonly AudioFileStorageSettings _audioStorageSettings; private readonly ILogger _logger; public ClearOrphanAudioJob(IEntryRepository entryRepository, IOptions storageSettings, - IOptions audioStorageSettings, ILoggerFactory logger) - { + IOptions audioStorageSettings, ILoggerFactory logger) { this._storageSettings = storageSettings.Value; this._audioStorageSettings = audioStorageSettings.Value; this._entryRepository = entryRepository; @@ -26,38 +23,28 @@ namespace PodNoms.Api.Services.Jobs this._logger = logger.CreateLogger(); } - public async Task Execute() - { - try - { + public async Task Execute() { + try { CloudStorageAccount storageAccount = CloudStorageAccount.Parse(_storageSettings.ConnectionString); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer container = blobClient.GetContainerReference(_audioStorageSettings.ContainerName); var blobs = await container.ListBlobsSegmentedAsync(null); - foreach (CloudBlockBlob blob in blobs.Results) - { - try - { + foreach (CloudBlockBlob blob in blobs.Results) { + try { Console.WriteLine(blob.StorageUri); var guid = blob.Name.Split('.')[0]; - if (!string.IsNullOrEmpty(guid)) - { + if (!string.IsNullOrEmpty(guid)) { var entry = await _entryRepository.GetByUidAsync(guid); - if (entry == null) - { + if (entry == null) { await blob.DeleteIfExistsAsync(); } } - } - catch (Exception e) - { + } catch (Exception e) { _logger.LogWarning($"Error processing blob {blob.Uri}\n{e.Message}"); } } - } - catch (Exception ex) - { + } catch (Exception ex) { _logger.LogError($"Error clearing orphans\n{ex.Message}"); } } diff --git a/server/Services/Processor/IUrlProcessService.cs b/server/Services/Processor/IUrlProcessService.cs index ad3b585..0319122 100644 --- a/server/Services/Processor/IUrlProcessService.cs +++ b/server/Services/Processor/IUrlProcessService.cs @@ -3,8 +3,9 @@ using PodNoms.Api.Models; namespace PodNoms.Api.Services.Processor { public interface IUrlProcessService { - Task GetInformation (int entryId); - Task GetInformation (PodcastEntry entry); - Task DownloadAudio (int entryId); + + Task GetInformation(int entryId); + Task GetInformation(PodcastEntry entry); + Task DownloadAudio(int entryId); } } \ No newline at end of file diff --git a/server/Services/Processor/UrlProcessService.cs b/server/Services/Processor/UrlProcessService.cs index ec74206..2a3e7e0 100644 --- a/server/Services/Processor/UrlProcessService.cs +++ b/server/Services/Processor/UrlProcessService.cs @@ -37,23 +37,23 @@ namespace PodNoms.Api.Services.Processor { uid, e); } - public async Task GetInformation(int entryId) { + public async Task GetInformation(int entryId) { var entry = await _repository.GetAsync(entryId); if (entry == null || string.IsNullOrEmpty(entry.SourceUrl)) { _logger.LogError("Unable to process item"); - return false; + return AudioType.Invalid; } if (entry.SourceUrl.EndsWith(".mp3") || entry.SourceUrl.EndsWith(".wav") || entry.SourceUrl.EndsWith(".aif")) { - return true; + return AudioType.Valid; } return await GetInformation(entry); } - public async Task GetInformation(PodcastEntry entry) { + public async Task GetInformation(PodcastEntry entry) { var downloader = new AudioDownloader(entry.SourceUrl, _applicationsSettings.Downloader); - await downloader.GetInfo(); - if (downloader.Properties != null) { + var ret = await downloader.GetInfo(); + if (ret == AudioType.Valid) { entry.Title = downloader.Properties?.Title; entry.Description = downloader.Properties?.Description; entry.ImageUrl = downloader.Properties?.Thumbnail; @@ -68,11 +68,8 @@ namespace PodNoms.Api.Services.Processor { _logger.LogDebug("***DOWNLOAD INFO RETRIEVED****\n"); _logger.LogDebug($"Title: {entry.Title}\nDescription: {entry.Description}\nAuthor: {entry.Author}\n"); - - // var pusherResult = await _sendProcessCompleteMessage(entry); - return true; } - return false; + return ret; } public async Task DownloadAudio(int entryId) { var entry = await _repository.GetAsync(entryId); diff --git a/server/Services/RemoteAudioTypes.cs b/server/Services/RemoteAudioTypes.cs new file mode 100644 index 0000000..36c0159 --- /dev/null +++ b/server/Services/RemoteAudioTypes.cs @@ -0,0 +1,7 @@ +namespace PodNoms.Api.Services { + public enum AudioType { + Invalid, + Valid, + Playlist + } +} \ No newline at end of file diff --git a/server/publish.sh b/server/publish.sh index 25d27b5..ce66001 100755 --- a/server/publish.sh +++ b/server/publish.sh @@ -3,3 +3,4 @@ unset DOCKER_HOST unset DOCKER_TLS_VERIFY docker build --rm -f Dockerfile -t fergalmoran/podnoms.api . && docker push fergalmoran/podnoms.api +docker push fergalmoran/podnoms.api