diff --git a/.gitignore b/.gitignore index dd2c5c4..b2bf6fd 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ certs/ promotion .working extension +client/tags diff --git a/client/.vscode/launch.json b/client/.vscode/launch.json index 8a08a79..fc7a729 100644 --- a/client/.vscode/launch.json +++ b/client/.vscode/launch.json @@ -4,16 +4,16 @@ { "type": "chrome", "request": "launch", - "name": "Launch Chrome with ng serve", - "url": "http://dev.podnoms.com:4200/", - "webRoot": "${workspaceRoot}" + "name": "Launch Chrome against localhost", + "url": "http://dev.podnoms.com:4200", + "webRoot": "${workspaceFolder}" }, { "type": "chrome", - "request": "launch", - "name": "Launch Chrome with ng test", - "url": "http://localhost:9876/debug.html", - "webRoot": "${workspaceRoot}" + "request": "attach", + "name": "Attach to Chrome", + "port": 9222, + "webRoot": "${workspaceFolder}" } ] -} \ No newline at end of file +} diff --git a/client/package-lock.json b/client/package-lock.json index fd42bb1..7280639 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -1,6 +1,6 @@ { "name": "pod-noms.web", - "version": "0.12.0", + "version": "0.13.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/client/src/app/components/podcast/podcast-add-url-form/podcast-add-url-form.component.ts b/client/src/app/components/podcast/podcast-add-url-form/podcast-add-url-form.component.ts index 0657c12..7e110f2 100644 --- a/client/src/app/components/podcast/podcast-add-url-form/podcast-add-url-form.component.ts +++ b/client/src/app/components/podcast/podcast-add-url-form/podcast-add-url-form.component.ts @@ -36,29 +36,22 @@ export class PodcastAddUrlFormComponent implements AfterViewInit { this.errorText = ''; if (this.isValidURL(urlToCheck)) { this.isPosting = true; - this._service.checkEntry(urlToCheck).subscribe( - r => { - if (r) { - const entry = new PodcastEntryModel( - this.podcast.id, - urlToCheck - ); - this.onUrlAddComplete.emit(entry); - this.isPosting = false; - this.newEntrySourceUrl = urlToCheck; - } else { - this.errorText = 'This is not a supported URL'; - this.isPosting = false; - this.newEntrySourceUrl = urlToCheck; - } - }, - err => { - this.errorText = 'This is not a supported URL'; - this.isPosting = false; - this.newEntrySourceUrl = urlToCheck; - } + const entry = new PodcastEntryModel( + this.podcast.id, + urlToCheck ); + this._service.addEntry(entry) + .subscribe(e => { + if (e) { + this.onUrlAddComplete.emit(e); + } + }, (err) => { + this.isPosting = false; + this.errorText = 'This does not look like a valid URL'; + this.newEntrySourceUrl = urlToCheck; + }); } else { + this.isPosting = false; this.errorText = 'This does not look like a valid URL'; this.newEntrySourceUrl = urlToCheck; } diff --git a/client/src/app/components/podcast/podcast.component.ts b/client/src/app/components/podcast/podcast.component.ts index c6129ff..e042ad1 100644 --- a/client/src/app/components/podcast/podcast.component.ts +++ b/client/src/app/components/podcast/podcast.component.ts @@ -82,6 +82,6 @@ export class PodcastComponent { } onUrlAddComplete(entry: PodcastEntryModel) { this.urlMode = false; - this._store.dispatch(new fromEntriesActions.AddAction(entry)); + this._store.dispatch(new fromEntriesActions.AddSuccessAction(entry)); } } diff --git a/client/src/app/effects/entries.effects.ts b/client/src/app/effects/entries.effects.ts index a32d9bb..11738e2 100644 --- a/client/src/app/effects/entries.effects.ts +++ b/client/src/app/effects/entries.effects.ts @@ -33,16 +33,18 @@ export class EntriesEffects { add$ = this.actions$ .ofType(entries.ADD) .switchMap((action: entries.AddAction) => - this._service.addEntry(action.payload) - ) - .map(res => { - console.log('EntriesEffects', 'add$', res); - return ({ type: entries.ADD_SUCCESS, payload: res }); - }) - .catch(err => { - console.error('EntriesEffects', 'add$', err); - return Observable.of({ type: entries.ADD_FAIL }); - }); + //this is probably (definitely) superfluous as we've now moved + //the addEntry into the component + this._service.updateEntry(action.payload) + .map(res => { + console.log('EntriesEffects', 'add$', res); + return ({ type: entries.ADD_SUCCESS, payload: res }); + }) + .catch(err => { + console.error('EntriesEffects', 'add$', err); + return Observable.of({ type: entries.ADD_FAIL }); + }) + ); @Effect() delete$ = this.actions$ .ofType(entries.DELETE) diff --git a/server/Controllers/EntryController.cs b/server/Controllers/EntryController.cs index b54575a..c022e76 100644 --- a/server/Controllers/EntryController.cs +++ b/server/Controllers/EntryController.cs @@ -15,7 +15,7 @@ using PodNoms.Api.Services.Processor; using PodNoms.Api.Services.Storage; namespace PodNoms.Api.Controllers { - [Route ("[controller]")] + [Route("[controller]")] public class EntryController : Controller { private readonly IPodcastRepository _podcastRepository; private readonly IEntryRepository _repository; @@ -26,12 +26,12 @@ namespace PodNoms.Api.Controllers { private readonly AudioFileStorageSettings _audioFileStorageSettings; private readonly StorageSettings _storageSettings; - public EntryController (IEntryRepository repository, + public EntryController(IEntryRepository repository, IPodcastRepository podcastRepository, IUnitOfWork unitOfWork, IMapper mapper, IOptions storageSettings, IOptions audioFileStorageSettings, IUrlProcessService processor, ILoggerFactory logger) { - this._logger = logger.CreateLogger (); + this._logger = logger.CreateLogger(); this._podcastRepository = podcastRepository; this._repository = repository; this._storageSettings = storageSettings.Value; @@ -41,83 +41,73 @@ namespace PodNoms.Api.Controllers { this._processor = processor; } - private void _processEntry (PodcastEntry entry) { + private void _processEntry(PodcastEntry entry) { try { - var infoJobId = BackgroundJob.Enqueue ( - service => service.GetInformation (entry.Id)); - var extract = BackgroundJob.ContinueWith ( - infoJobId, service => service.DownloadAudio (entry.Id)); - var upload = BackgroundJob.ContinueWith ( - extract, service => service.UploadAudio (entry.Id, entry.AudioUrl)); + var extractJobId = BackgroundJob.Enqueue( + service => service.DownloadAudio(entry.Id)); + var upload = BackgroundJob.ContinueWith( + extractJobId, service => service.UploadAudio(entry.Id, entry.AudioUrl)); } catch (InvalidOperationException ex) { - _logger.LogError ($"Failed submitting job to processor\n{ex.Message}"); + _logger.LogError($"Failed submitting job to processor\n{ex.Message}"); entry.ProcessingStatus = ProcessingStatus.Failed; } } - [HttpGet ("all/{podcastSlug}")] - public async Task GetAllForSlug (string podcastSlug) { - var entries = await _repository.GetAllAsync (podcastSlug); - var results = _mapper.Map, List> (entries.ToList ()); + [HttpGet("all/{podcastSlug}")] + public async Task GetAllForSlug(string podcastSlug) { + var entries = await _repository.GetAllAsync(podcastSlug); + var results = _mapper.Map, List>(entries.ToList()); - return Ok (results); + return Ok(results); } [HttpPost] - public async Task Post ([FromBody] PodcastEntryViewModel item) { + public async Task> Post([FromBody] PodcastEntryViewModel item) { // first check url is valid - - var entry = _mapper.Map (item); - if (entry.ProcessingStatus == ProcessingStatus.Accepted) { - var podcast = await _podcastRepository.GetAsync (item.PodcastId); - entry.ImageUrl = $"{_storageSettings.CdnUrl}static/images/default-entry.png"; - - entry.Podcast = podcast; - entry.Processed = false; - if (string.IsNullOrEmpty (item.Title)) { - entry.Title = "Waiting for information"; - } else { - entry.Title = item.Title; + var entry = _mapper.Map(item); + var podcast = await _podcastRepository.GetAsync(item.PodcastId); + if (podcast != null && await _processor.GetInformation(entry)) { + if (entry.ProcessingStatus == ProcessingStatus.Processing) { + entry.Podcast = podcast; + entry.Processed = false; + await _repository.AddOrUpdateAsync(entry); + await _unitOfWork.CompleteAsync(); + _processEntry(entry); + var result = _mapper.Map(entry); + return result; } } - await _repository.AddOrUpdateAsync (entry); - await _unitOfWork.CompleteAsync (); - - if (entry.ProcessingStatus.Equals (ProcessingStatus.Accepted) && entry.Id != 0) { - _processEntry (entry); - } - var result = _mapper.Map (entry); - return Ok (result); - + return BadRequest(); } - [HttpDelete ("{id}")] - public async Task Delete (int id) { - await this._repository.DeleteAsync (id); - await _unitOfWork.CompleteAsync (); - return Ok (); + [HttpDelete("{id}")] + public async Task Delete(int id) { + await this._repository.DeleteAsync(id); + await _unitOfWork.CompleteAsync(); + return Ok(); } - - [HttpPost ("isvalid")] - public async Task IsValid ([FromBody] string url) { - if (!string.IsNullOrEmpty (url)) { - var isValid = await _processor.CheckUrlValid (url); - if (isValid) return Ok (); - } - return BadRequest (); - } - - [HttpPost ("resubmit")] - public async Task ReSubmit ([FromBody] PodcastEntryViewModel item) { - var entry = await _repository.GetAsync (item.Id); + [HttpPost("/preprocess")] + public async Task> PreProcess(PodcastEntryViewModel item) { + var entry = await _repository.GetAsync(item.Id); + entry.ProcessingStatus = ProcessingStatus.Accepted; + var response = _processor.GetInformation(item.Id); entry.ProcessingStatus = ProcessingStatus.Processing; - await _unitOfWork.CompleteAsync (); - if (entry.ProcessingStatus != ProcessingStatus.Processed) { - _processEntry (entry); - } + await _unitOfWork.CompleteAsync(); - return Ok (entry); + var result = _mapper.Map(entry); + return result; + } + + [HttpPost("resubmit")] + public async Task ReSubmit([FromBody] PodcastEntryViewModel item) { + var entry = await _repository.GetAsync(item.Id); + entry.ProcessingStatus = ProcessingStatus.Processing; + await _unitOfWork.CompleteAsync(); + if (entry.ProcessingStatus != ProcessingStatus.Processed) { + _processEntry(entry); + } + return Ok(entry); } } -} \ No newline at end of file +} diff --git a/server/Persistence/EntryRepository.cs b/server/Persistence/EntryRepository.cs index 976d30e..1d78cbc 100644 --- a/server/Persistence/EntryRepository.cs +++ b/server/Persistence/EntryRepository.cs @@ -5,64 +5,51 @@ using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using PodNoms.Api.Models; -namespace PodNoms.Api.Persistence -{ - public class EntryRepository : IEntryRepository - { +namespace PodNoms.Api.Persistence { + public class EntryRepository : IEntryRepository { private readonly PodnomsDbContext _context; - public EntryRepository(PodnomsDbContext context) - { + public EntryRepository(PodnomsDbContext context) { this._context = context; } - public async Task GetAsync(int id) - { + public async Task GetAsync(int id) { var entry = await _context.PodcastEntries .Include(e => e.Podcast) .Include(e => e.Podcast.User) .SingleOrDefaultAsync(e => e.Id == id); return entry; } - public async Task GetByUidAsync(string uid) - { + public async Task GetByUidAsync(string uid) { var entry = await _context.PodcastEntries .SingleOrDefaultAsync(e => e.Uid == uid); return entry; } - public async Task> GetAllAsync(int podcastId) - { + public async Task> GetAllAsync(int podcastId) { var entries = await _context.PodcastEntries .AsNoTracking() .Where(e => e.PodcastId == podcastId) .ToListAsync(); return entries; } - public async Task> GetAllAsync(string podcastSlug) - { + public async Task> GetAllAsync(string podcastSlug) { var entries = await _context.PodcastEntries .Where(e => e.Podcast.Slug == podcastSlug) .ToListAsync(); return entries; } - public async Task AddOrUpdateAsync(PodcastEntry entry) - { - if (entry.Id != 0) - { + public async Task AddOrUpdateAsync(PodcastEntry entry) { + if (entry.Id != 0) { // _context.Entry(entry).State = EntityState.Modified _context.PodcastEntries.Attach(entry); _context.Entry(entry).State = EntityState.Modified; - } - else - { + } else { if (string.IsNullOrEmpty(entry.Uid)) entry.Uid = System.Guid.NewGuid().ToString(); await _context.PodcastEntries.AddAsync(entry); - entry.ProcessingStatus = ProcessingStatus.Accepted; } return entry; } - public async Task DeleteAsync(int id) - { + public async Task DeleteAsync(int id) { var entry = await GetAsync(id); _context.Remove(entry); } diff --git a/server/Services/Downloader/AudioDownloader.cs b/server/Services/Downloader/AudioDownloader.cs index 436145b..b0e96c0 100644 --- a/server/Services/Downloader/AudioDownloader.cs +++ b/server/Services/Downloader/AudioDownloader.cs @@ -15,7 +15,7 @@ namespace PodNoms.Api.Services.Downloader { public class AudioDownloader { private readonly string _url; private readonly string _downloader; - public dynamic Properties { get; private set; } + public VideoDownloadInfo Properties { get; private set; } protected const string DOWNLOADRATESTRING = "iB/s"; protected const string DOWNLOADSIZESTRING = "iB"; protected const string ETASTRING = "ETA"; @@ -54,7 +54,7 @@ namespace PodNoms.Api.Services.Downloader { await Task.Run(() => { var youtubeDl = new YoutubeDL(); youtubeDl.VideoUrl = this._url; - this.Properties = youtubeDl.GetDownloadInfo(); + this.Properties = youtubeDl.GetDownloadInfo() as VideoDownloadInfo; var info = youtubeDl.GetDownloadInfo(); ret = ( info != null && diff --git a/server/Services/Processor/AudioUploadProcessService.cs b/server/Services/Processor/AudioUploadProcessService.cs index ef131ed..db230f8 100644 --- a/server/Services/Processor/AudioUploadProcessService.cs +++ b/server/Services/Processor/AudioUploadProcessService.cs @@ -36,7 +36,7 @@ namespace PodNoms.Api.Services.Processor { entry.ProcessingStatus = ProcessingStatus.Uploading; await _unitOfWork.CompleteAsync (); try { - // bit messy but can't figure how to pass youtube-dl job result to this job + // bit messy but can't figure how to p ass youtube-dl job result to this job // so using AudioUrl as a proxy if (string.IsNullOrEmpty (localFile)) localFile = entry.AudioUrl; diff --git a/server/Services/Processor/IUrlProcessService.cs b/server/Services/Processor/IUrlProcessService.cs index f8943cd..ad3b585 100644 --- a/server/Services/Processor/IUrlProcessService.cs +++ b/server/Services/Processor/IUrlProcessService.cs @@ -1,9 +1,10 @@ using System.Threading.Tasks; +using PodNoms.Api.Models; namespace PodNoms.Api.Services.Processor { public interface IUrlProcessService { - Task CheckUrlValid (string url); Task GetInformation (int entryId); + Task GetInformation (PodcastEntry entry); Task DownloadAudio (int entryId); } } \ No newline at end of file diff --git a/server/Services/Processor/ProcessService.cs b/server/Services/Processor/ProcessService.cs index fd79fc0..8c5fdbc 100644 --- a/server/Services/Processor/ProcessService.cs +++ b/server/Services/Processor/ProcessService.cs @@ -9,47 +9,37 @@ using PodNoms.Api.Models; using PodNoms.Api.Models.ViewModels; using PodNoms.Api.Services.Realtime; -namespace PodNoms.Api.Services.Processor -{ - internal class ProcessService - { +namespace PodNoms.Api.Services.Processor { + internal class ProcessService { protected readonly ILogger _logger; protected readonly IRealTimeUpdater _realtime; protected readonly IMapper _mapper; protected readonly JsonSerializer _serializer; - protected ProcessService(ILoggerFactory logger, IMapper mapper, IRealTimeUpdater pusher) - { + protected ProcessService(ILoggerFactory logger, IMapper mapper, IRealTimeUpdater pusher) { this._logger = logger.CreateLogger(); this._realtime = pusher; this._mapper = mapper; - this._serializer = new JsonSerializer - { + this._serializer = new JsonSerializer { ContractResolver = new CamelCasePropertyNamesContractResolver() }; } - protected async Task _sendProcessCompleteMessage(PodcastEntry entry) - { + protected async Task _sendProcessCompleteMessage(PodcastEntry entry) { var result = _mapper.Map(entry); return await _sendProcessUpdate(entry.Podcast.User.GetUserId(), entry.Uid, "info_processed", result); } - protected async Task _sendProgressUpdate(string userId, string itemUid, ProcessProgressEvent data) - { + protected async Task _sendProgressUpdate(string userId, string itemUid, ProcessProgressEvent data) { return await _realtime.SendProcessUpdate(userId, itemUid, "progress_update", data); } - protected async Task _sendProcessUpdate(string userId, string itemUid, string message, PodcastEntryViewModel data) - { - try - { + protected async Task _sendProcessUpdate(string userId, string itemUid, string message, PodcastEntryViewModel data) { + try { return await _realtime.SendProcessUpdate( userId, itemUid, message, data); - } - catch (Exception ex) - { + } catch (Exception ex) { _logger.LogError(123456, ex, "Error sending realtime message"); } return false; diff --git a/server/Services/Processor/UrlProcessService.cs b/server/Services/Processor/UrlProcessService.cs index f2b2588..ec74206 100644 --- a/server/Services/Processor/UrlProcessService.cs +++ b/server/Services/Processor/UrlProcessService.cs @@ -23,86 +23,85 @@ namespace PodNoms.Api.Services.Processor { public ApplicationsSettings _applicationsSettings { get; } - public UrlProcessService (IEntryRepository repository, IUnitOfWork unitOfWork, + public UrlProcessService(IEntryRepository repository, IUnitOfWork unitOfWork, IFileUploader fileUploader, IOptions applicationsSettings, - ILoggerFactory logger, IMapper mapper, IRealTimeUpdater pusher) : base (logger, mapper, pusher) { + ILoggerFactory logger, IMapper mapper, IRealTimeUpdater pusher) : base(logger, mapper, pusher) { this._applicationsSettings = applicationsSettings.Value; this._repository = repository; this._unitOfWork = unitOfWork; } - private async Task __downloader_progress (string userId, string uid, ProcessProgressEvent e) { - await _sendProgressUpdate ( + private async Task __downloader_progress(string userId, string uid, ProcessProgressEvent e) { + await _sendProgressUpdate( userId, uid, e); } - - public async Task CheckUrlValid (string url) { - return await new AudioDownloader (url, _applicationsSettings.Downloader) - .CheckUrlValid (); - } - - public async Task GetInformation (int entryId) { - var entry = await _repository.GetAsync (entryId); - if (entry == null || string.IsNullOrEmpty (entry.SourceUrl)) { - _logger.LogError ("Unable to process item"); + public async Task GetInformation(int entryId) { + var entry = await _repository.GetAsync(entryId); + if (entry == null || string.IsNullOrEmpty(entry.SourceUrl)) { + _logger.LogError("Unable to process item"); return false; } - if (entry.SourceUrl.EndsWith (".mp3") || entry.SourceUrl.EndsWith (".wav") || entry.SourceUrl.EndsWith (".aif")) { + if (entry.SourceUrl.EndsWith(".mp3") || entry.SourceUrl.EndsWith(".wav") || entry.SourceUrl.EndsWith(".aif")) { return true; } - var downloader = new AudioDownloader (entry.SourceUrl, _applicationsSettings.Downloader); - downloader.DownloadInfo (); + return await GetInformation(entry); + } + + public async Task GetInformation(PodcastEntry entry) { + + var downloader = new AudioDownloader(entry.SourceUrl, _applicationsSettings.Downloader); + await downloader.GetInfo(); if (downloader.Properties != null) { - entry.Title = downloader.Properties?.title; - entry.Description = downloader.Properties?.description; - entry.ImageUrl = downloader.Properties?.thumbnail; + entry.Title = downloader.Properties?.Title; + entry.Description = downloader.Properties?.Description; + entry.ImageUrl = downloader.Properties?.Thumbnail; entry.ProcessingStatus = ProcessingStatus.Processing; try { - entry.Author = downloader.Properties?.uploader; + entry.Author = downloader.Properties?.Uploader; } catch (Exception) { - _logger.LogWarning ($"Unable to extract downloader info for: {entry.SourceUrl}"); + _logger.LogWarning($"Unable to extract downloader info for: {entry.SourceUrl}"); } - await _unitOfWork.CompleteAsync (); + await _unitOfWork.CompleteAsync(); - _logger.LogDebug ("***DOWNLOAD INFO RETRIEVED****\n"); - _logger.LogDebug ($"Title: {entry.Title}\nDescription: {entry.Description}\nAuthor: {entry.Author}\n"); + _logger.LogDebug("***DOWNLOAD INFO RETRIEVED****\n"); + _logger.LogDebug($"Title: {entry.Title}\nDescription: {entry.Description}\nAuthor: {entry.Author}\n"); - var pusherResult = await _sendProcessCompleteMessage (entry); + // var pusherResult = await _sendProcessCompleteMessage(entry); return true; } return false; } - public async Task DownloadAudio (int entryId) { - var entry = await _repository.GetAsync (entryId); + public async Task DownloadAudio(int entryId) { + var entry = await _repository.GetAsync(entryId); if (entry == null) return false; try { - var downloader = new AudioDownloader (entry.SourceUrl, _applicationsSettings.Downloader); + var downloader = new AudioDownloader(entry.SourceUrl, _applicationsSettings.Downloader); var outputFile = - Path.Combine (System.IO.Path.GetTempPath (), $"{System.Guid.NewGuid().ToString()}.mp3"); + Path.Combine(System.IO.Path.GetTempPath(), $"{System.Guid.NewGuid().ToString()}.mp3"); - downloader.DownloadProgress += async (s, e) => await __downloader_progress (entry.Podcast.User.GetUserId (), entry.Uid, e); + downloader.DownloadProgress += async (s, e) => await __downloader_progress(entry.Podcast.User.GetUserId(), entry.Uid, e); downloader.PostProcessing += (s, e) => { - Console.WriteLine (e); + Console.WriteLine(e); }; - var sourceFile = downloader.DownloadAudio (entry.Uid); - if (!string.IsNullOrEmpty (sourceFile)) { + var sourceFile = downloader.DownloadAudio(entry.Uid); + if (!string.IsNullOrEmpty(sourceFile)) { entry.ProcessingStatus = ProcessingStatus.Uploading; entry.AudioUrl = sourceFile; - await _sendProcessCompleteMessage (entry); - await _unitOfWork.CompleteAsync (); + await _sendProcessCompleteMessage(entry); + await _unitOfWork.CompleteAsync(); } } catch (Exception ex) { - _logger.LogError ($"Entry: {entryId}\n{ex.Message}"); + _logger.LogError($"Entry: {entryId}\n{ex.Message}"); entry.ProcessingStatus = ProcessingStatus.Failed; entry.ProcessingPayload = ex.Message; - await _unitOfWork.CompleteAsync (); - await _sendProcessCompleteMessage (entry); + await _unitOfWork.CompleteAsync(); + await _sendProcessCompleteMessage(entry); } return false; }