diff --git a/client/src/app/components/debug/debug.component.html b/client/src/app/components/debug/debug.component.html index 16fd8f9..237ef06 100644 --- a/client/src/app/components/debug/debug.component.html +++ b/client/src/app/components/debug/debug.component.html @@ -23,6 +23,7 @@
+
diff --git a/client/src/app/components/debug/debug.component.ts b/client/src/app/components/debug/debug.component.ts index 9fc88d2..c9b92c5 100644 --- a/client/src/app/components/debug/debug.component.ts +++ b/client/src/app/components/debug/debug.component.ts @@ -54,6 +54,13 @@ export class DebugComponent implements OnInit { console.log('debug.component.ts', 'processPlaylists', e) ); } + processPlaylistItems() { + this._jobsService + .processPlaylistItems() + .subscribe((e) => + console.log('debug.component.ts', 'processPlaylists', e) + ); + } updateYouTubeDl() { this._jobsService .updateYouTubeDl() diff --git a/client/src/app/services/jobs.service.ts b/client/src/app/services/jobs.service.ts index d1f27be..5d61f88 100644 --- a/client/src/app/services/jobs.service.ts +++ b/client/src/app/services/jobs.service.ts @@ -18,6 +18,11 @@ export class JobsService { environment.API_HOST + '/job/processplaylists' ); } + processPlaylistItems(): Observable { + return this._http.get( + environment.API_HOST + '/job/processplaylistitems' + ); + } updateYouTubeDl(): Observable { return this._http.get( environment.API_HOST + '/job/updateyoutubedl' diff --git a/server/Controllers/EntryController.cs b/server/Controllers/EntryController.cs index 849a60a..6b44fb0 100644 --- a/server/Controllers/EntryController.cs +++ b/server/Controllers/EntryController.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Threading.Tasks; using AutoMapper; using Hangfire; +using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Identity; using Microsoft.AspNetCore.Mvc; @@ -32,6 +33,7 @@ namespace PodNoms.Api.Controllers { private readonly IUnitOfWork _unitOfWork; private readonly IMapper _mapper; private readonly IUrlProcessService _processor; + private readonly IHostingEnvironment _hostingEnvironment; private readonly ILogger _logger; private readonly AudioFileStorageSettings _audioFileStorageSettings; private readonly StorageSettings _storageSettings; @@ -43,6 +45,7 @@ namespace PodNoms.Api.Controllers { IConfiguration options, IUrlProcessService processor, ILoggerFactory logger, UserManager userManager, + IHostingEnvironment hostingEnvironment, IHttpContextAccessor contextAccessor) : base(contextAccessor, userManager) { this._logger = logger.CreateLogger(); this._podcastRepository = podcastRepository; @@ -53,6 +56,7 @@ namespace PodNoms.Api.Controllers { this._audioFileStorageSettings = audioFileStorageSettings.Value; this._mapper = mapper; this._processor = processor; + this._hostingEnvironment = hostingEnvironment; } private void _processEntry(PodcastEntry entry) { @@ -111,7 +115,7 @@ namespace PodNoms.Api.Controllers { var result = _mapper.Map(entry); return result; } - } else if (status == AudioType.Playlist) { + } else if (status == AudioType.Playlist && _hostingEnvironment.IsDevelopment()) { entry.ProcessingStatus = ProcessingStatus.Deferred; return Accepted(entry); } diff --git a/server/Controllers/JobController.cs b/server/Controllers/JobController.cs index 8db8431..e6c874e 100644 --- a/server/Controllers/JobController.cs +++ b/server/Controllers/JobController.cs @@ -19,6 +19,11 @@ namespace PodNoms.Api.Controllers { var infoJobId = BackgroundJob.Enqueue(service => service.Execute()); return Ok(); } + [HttpGet("processplaylistitems")] + public IActionResult ProcessPlaylistItems() { + var infoJobId = BackgroundJob.Enqueue(service => service.Execute()); + return Ok(); + } [HttpGet("updateyoutubedl")] public IActionResult UpdateYouTubeDl() { var infoJobId = BackgroundJob.Enqueue(service => service.Execute()); diff --git a/server/Migrations/20180507153433_ParsedPlaylistVideos.Designer.cs b/server/Migrations/20180507153433_ParsedPlaylistVideos.Designer.cs new file mode 100644 index 0000000..f658378 --- /dev/null +++ b/server/Migrations/20180507153433_ParsedPlaylistVideos.Designer.cs @@ -0,0 +1,390 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Migrations; +using PodNoms.Api.Persistence; + +namespace PodNoms.Api.Migrations +{ + [DbContext(typeof(PodnomsDbContext))] + [Migration("20180507153433_ParsedPlaylistVideos")] + partial class ParsedPlaylistVideos + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "2.1.0-preview2-30571") + .HasAnnotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRole", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken(); + + b.Property("Name") + .HasMaxLength(256); + + b.Property("NormalizedName") + .HasMaxLength(256); + + b.HasKey("Id"); + + b.HasIndex("NormalizedName") + .IsUnique() + .HasName("RoleNameIndex") + .HasFilter("[NormalizedName] IS NOT NULL"); + + b.ToTable("AspNetRoles"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("ClaimType"); + + b.Property("ClaimValue"); + + b.Property("RoleId") + .IsRequired(); + + b.HasKey("Id"); + + b.HasIndex("RoleId"); + + b.ToTable("AspNetRoleClaims"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("ClaimType"); + + b.Property("ClaimValue"); + + b.Property("UserId") + .IsRequired(); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.ToTable("AspNetUserClaims"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.Property("LoginProvider"); + + b.Property("ProviderKey"); + + b.Property("ProviderDisplayName"); + + b.Property("UserId") + .IsRequired(); + + b.HasKey("LoginProvider", "ProviderKey"); + + b.HasIndex("UserId"); + + b.ToTable("AspNetUserLogins"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole", b => + { + b.Property("UserId"); + + b.Property("RoleId"); + + b.HasKey("UserId", "RoleId"); + + b.HasIndex("RoleId"); + + b.ToTable("AspNetUserRoles"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.Property("UserId"); + + b.Property("LoginProvider"); + + b.Property("Name"); + + b.Property("Value"); + + b.HasKey("UserId", "LoginProvider", "Name"); + + b.ToTable("AspNetUserTokens"); + }); + + modelBuilder.Entity("PodNoms.Api.Models.ParsedPlaylistVideo", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("CreateDate"); + + b.Property("IsProcessed"); + + b.Property("PlaylistId"); + + b.Property("UpdateDate"); + + b.Property("VideoId"); + + b.HasKey("Id"); + + b.HasIndex("PlaylistId"); + + b.ToTable("ParsedPlaylistVideos"); + }); + + modelBuilder.Entity("PodNoms.Api.Models.Playlist", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("CreateDate"); + + b.Property("PodcastId"); + + b.Property("SourceUrl"); + + b.Property("UpdateDate"); + + b.HasKey("Id"); + + b.HasIndex("PodcastId"); + + b.ToTable("Playlists"); + }); + + modelBuilder.Entity("PodNoms.Api.Models.Podcast", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("AppUserId"); + + b.Property("CreateDate"); + + b.Property("Description"); + + b.Property("Slug"); + + b.Property("TemporaryImageUrl"); + + b.Property("Title"); + + b.Property("Uid"); + + b.Property("UpdateDate"); + + b.HasKey("Id"); + + b.HasIndex("AppUserId"); + + b.ToTable("Podcasts"); + }); + + modelBuilder.Entity("PodNoms.Api.Models.PodcastEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("AudioFileSize"); + + b.Property("AudioLength"); + + b.Property("AudioUrl"); + + b.Property("Author"); + + b.Property("CreateDate"); + + b.Property("Description"); + + b.Property("ImageUrl"); + + b.Property("PlaylistId"); + + b.Property("PodcastId"); + + b.Property("Processed"); + + b.Property("ProcessingPayload"); + + b.Property("ProcessingStatus"); + + b.Property("SourceUrl"); + + b.Property("Title"); + + b.Property("Uid"); + + b.Property("UpdateDate"); + + b.HasKey("Id"); + + b.HasIndex("PlaylistId"); + + b.HasIndex("PodcastId"); + + b.ToTable("PodcastEntries"); + }); + + modelBuilder.Entity("PodNoms.Api.Services.Auth.ApplicationUser", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("AccessFailedCount"); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken(); + + b.Property("Email") + .HasMaxLength(256); + + b.Property("EmailConfirmed"); + + b.Property("FacebookId"); + + b.Property("FirstName"); + + b.Property("LastName"); + + b.Property("LockoutEnabled"); + + b.Property("LockoutEnd"); + + b.Property("NormalizedEmail") + .HasMaxLength(256); + + b.Property("NormalizedUserName") + .HasMaxLength(256); + + b.Property("PasswordHash"); + + b.Property("PhoneNumber"); + + b.Property("PhoneNumberConfirmed"); + + b.Property("PictureUrl"); + + b.Property("SecurityStamp"); + + b.Property("Slug"); + + b.Property("TwoFactorEnabled"); + + b.Property("UserName") + .HasMaxLength(256); + + b.HasKey("Id"); + + b.HasIndex("NormalizedEmail") + .HasName("EmailIndex"); + + b.HasIndex("NormalizedUserName") + .IsUnique() + .HasName("UserNameIndex") + .HasFilter("[NormalizedUserName] IS NOT NULL"); + + b.ToTable("AspNetUsers"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole") + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.HasOne("PodNoms.Api.Services.Auth.ApplicationUser") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.HasOne("PodNoms.Api.Services.Auth.ApplicationUser") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole") + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("PodNoms.Api.Services.Auth.ApplicationUser") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.HasOne("PodNoms.Api.Services.Auth.ApplicationUser") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("PodNoms.Api.Models.ParsedPlaylistVideo", b => + { + b.HasOne("PodNoms.Api.Models.Playlist", "Playlist") + .WithMany("ParsedPlaylistVideos") + .HasForeignKey("PlaylistId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("PodNoms.Api.Models.Playlist", b => + { + b.HasOne("PodNoms.Api.Models.Podcast", "Podcast") + .WithMany() + .HasForeignKey("PodcastId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("PodNoms.Api.Models.Podcast", b => + { + b.HasOne("PodNoms.Api.Services.Auth.ApplicationUser", "AppUser") + .WithMany() + .HasForeignKey("AppUserId"); + }); + + modelBuilder.Entity("PodNoms.Api.Models.PodcastEntry", b => + { + b.HasOne("PodNoms.Api.Models.Playlist") + .WithMany("PodcastEntries") + .HasForeignKey("PlaylistId"); + + b.HasOne("PodNoms.Api.Models.Podcast", "Podcast") + .WithMany("PodcastEntries") + .HasForeignKey("PodcastId") + .OnDelete(DeleteBehavior.Cascade); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/server/Migrations/20180507153433_ParsedPlaylistVideos.cs b/server/Migrations/20180507153433_ParsedPlaylistVideos.cs new file mode 100644 index 0000000..8c11030 --- /dev/null +++ b/server/Migrations/20180507153433_ParsedPlaylistVideos.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Generic; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace PodNoms.Api.Migrations +{ + public partial class ParsedPlaylistVideos : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterColumn( + name: "CreateDate", + table: "Podcasts", + nullable: false, + oldClrType: typeof(DateTime), + oldDefaultValueSql: "getdate()"); + + migrationBuilder.AlterColumn( + name: "CreateDate", + table: "PodcastEntries", + nullable: false, + oldClrType: typeof(DateTime), + oldDefaultValueSql: "getdate()"); + + migrationBuilder.CreateTable( + name: "ParsedPlaylistVideos", + columns: table => new + { + CreateDate = table.Column(nullable: false), + UpdateDate = table.Column(nullable: false), + Id = table.Column(nullable: false) + .Annotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn), + VideoId = table.Column(nullable: true), + IsProcessed = table.Column(nullable: false), + PlaylistId = table.Column(nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_ParsedPlaylistVideos", x => x.Id); + table.ForeignKey( + name: "FK_ParsedPlaylistVideos_Playlists_PlaylistId", + column: x => x.PlaylistId, + principalTable: "Playlists", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "IX_ParsedPlaylistVideos_PlaylistId", + table: "ParsedPlaylistVideos", + column: "PlaylistId"); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "ParsedPlaylistVideos"); + + migrationBuilder.AlterColumn( + name: "CreateDate", + table: "Podcasts", + nullable: false, + defaultValueSql: "getdate()", + oldClrType: typeof(DateTime)); + + migrationBuilder.AlterColumn( + name: "CreateDate", + table: "PodcastEntries", + nullable: false, + defaultValueSql: "getdate()", + oldClrType: typeof(DateTime)); + } + } +} diff --git a/server/Migrations/20180507155436_Rename_ParsedPlaylistVideos.Designer.cs b/server/Migrations/20180507155436_Rename_ParsedPlaylistVideos.Designer.cs new file mode 100644 index 0000000..387ba8d --- /dev/null +++ b/server/Migrations/20180507155436_Rename_ParsedPlaylistVideos.Designer.cs @@ -0,0 +1,390 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Migrations; +using PodNoms.Api.Persistence; + +namespace PodNoms.Api.Migrations +{ + [DbContext(typeof(PodnomsDbContext))] + [Migration("20180507155436_Rename_ParsedPlaylistVideos")] + partial class Rename_ParsedPlaylistVideos + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "2.1.0-preview2-30571") + .HasAnnotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRole", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken(); + + b.Property("Name") + .HasMaxLength(256); + + b.Property("NormalizedName") + .HasMaxLength(256); + + b.HasKey("Id"); + + b.HasIndex("NormalizedName") + .IsUnique() + .HasName("RoleNameIndex") + .HasFilter("[NormalizedName] IS NOT NULL"); + + b.ToTable("AspNetRoles"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("ClaimType"); + + b.Property("ClaimValue"); + + b.Property("RoleId") + .IsRequired(); + + b.HasKey("Id"); + + b.HasIndex("RoleId"); + + b.ToTable("AspNetRoleClaims"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("ClaimType"); + + b.Property("ClaimValue"); + + b.Property("UserId") + .IsRequired(); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.ToTable("AspNetUserClaims"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.Property("LoginProvider"); + + b.Property("ProviderKey"); + + b.Property("ProviderDisplayName"); + + b.Property("UserId") + .IsRequired(); + + b.HasKey("LoginProvider", "ProviderKey"); + + b.HasIndex("UserId"); + + b.ToTable("AspNetUserLogins"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole", b => + { + b.Property("UserId"); + + b.Property("RoleId"); + + b.HasKey("UserId", "RoleId"); + + b.HasIndex("RoleId"); + + b.ToTable("AspNetUserRoles"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.Property("UserId"); + + b.Property("LoginProvider"); + + b.Property("Name"); + + b.Property("Value"); + + b.HasKey("UserId", "LoginProvider", "Name"); + + b.ToTable("AspNetUserTokens"); + }); + + modelBuilder.Entity("PodNoms.Api.Models.ParsedPlaylistItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("CreateDate"); + + b.Property("IsProcessed"); + + b.Property("PlaylistId"); + + b.Property("UpdateDate"); + + b.Property("VideoId"); + + b.HasKey("Id"); + + b.HasIndex("PlaylistId"); + + b.ToTable("ParsedPlaylistItems"); + }); + + modelBuilder.Entity("PodNoms.Api.Models.Playlist", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("CreateDate"); + + b.Property("PodcastId"); + + b.Property("SourceUrl"); + + b.Property("UpdateDate"); + + b.HasKey("Id"); + + b.HasIndex("PodcastId"); + + b.ToTable("Playlists"); + }); + + modelBuilder.Entity("PodNoms.Api.Models.Podcast", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("AppUserId"); + + b.Property("CreateDate"); + + b.Property("Description"); + + b.Property("Slug"); + + b.Property("TemporaryImageUrl"); + + b.Property("Title"); + + b.Property("Uid"); + + b.Property("UpdateDate"); + + b.HasKey("Id"); + + b.HasIndex("AppUserId"); + + b.ToTable("Podcasts"); + }); + + modelBuilder.Entity("PodNoms.Api.Models.PodcastEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("AudioFileSize"); + + b.Property("AudioLength"); + + b.Property("AudioUrl"); + + b.Property("Author"); + + b.Property("CreateDate"); + + b.Property("Description"); + + b.Property("ImageUrl"); + + b.Property("PlaylistId"); + + b.Property("PodcastId"); + + b.Property("Processed"); + + b.Property("ProcessingPayload"); + + b.Property("ProcessingStatus"); + + b.Property("SourceUrl"); + + b.Property("Title"); + + b.Property("Uid"); + + b.Property("UpdateDate"); + + b.HasKey("Id"); + + b.HasIndex("PlaylistId"); + + b.HasIndex("PodcastId"); + + b.ToTable("PodcastEntries"); + }); + + modelBuilder.Entity("PodNoms.Api.Services.Auth.ApplicationUser", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("AccessFailedCount"); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken(); + + b.Property("Email") + .HasMaxLength(256); + + b.Property("EmailConfirmed"); + + b.Property("FacebookId"); + + b.Property("FirstName"); + + b.Property("LastName"); + + b.Property("LockoutEnabled"); + + b.Property("LockoutEnd"); + + b.Property("NormalizedEmail") + .HasMaxLength(256); + + b.Property("NormalizedUserName") + .HasMaxLength(256); + + b.Property("PasswordHash"); + + b.Property("PhoneNumber"); + + b.Property("PhoneNumberConfirmed"); + + b.Property("PictureUrl"); + + b.Property("SecurityStamp"); + + b.Property("Slug"); + + b.Property("TwoFactorEnabled"); + + b.Property("UserName") + .HasMaxLength(256); + + b.HasKey("Id"); + + b.HasIndex("NormalizedEmail") + .HasName("EmailIndex"); + + b.HasIndex("NormalizedUserName") + .IsUnique() + .HasName("UserNameIndex") + .HasFilter("[NormalizedUserName] IS NOT NULL"); + + b.ToTable("AspNetUsers"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole") + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.HasOne("PodNoms.Api.Services.Auth.ApplicationUser") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.HasOne("PodNoms.Api.Services.Auth.ApplicationUser") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole") + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("PodNoms.Api.Services.Auth.ApplicationUser") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.HasOne("PodNoms.Api.Services.Auth.ApplicationUser") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("PodNoms.Api.Models.ParsedPlaylistItem", b => + { + b.HasOne("PodNoms.Api.Models.Playlist", "Playlist") + .WithMany("ParsedPlaylistItems") + .HasForeignKey("PlaylistId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("PodNoms.Api.Models.Playlist", b => + { + b.HasOne("PodNoms.Api.Models.Podcast", "Podcast") + .WithMany() + .HasForeignKey("PodcastId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("PodNoms.Api.Models.Podcast", b => + { + b.HasOne("PodNoms.Api.Services.Auth.ApplicationUser", "AppUser") + .WithMany() + .HasForeignKey("AppUserId"); + }); + + modelBuilder.Entity("PodNoms.Api.Models.PodcastEntry", b => + { + b.HasOne("PodNoms.Api.Models.Playlist") + .WithMany("PodcastEntries") + .HasForeignKey("PlaylistId"); + + b.HasOne("PodNoms.Api.Models.Podcast", "Podcast") + .WithMany("PodcastEntries") + .HasForeignKey("PodcastId") + .OnDelete(DeleteBehavior.Cascade); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/server/Migrations/20180507155436_Rename_ParsedPlaylistVideos.cs b/server/Migrations/20180507155436_Rename_ParsedPlaylistVideos.cs new file mode 100644 index 0000000..b752eec --- /dev/null +++ b/server/Migrations/20180507155436_Rename_ParsedPlaylistVideos.cs @@ -0,0 +1,78 @@ +using System; +using System.Collections.Generic; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace PodNoms.Api.Migrations +{ + public partial class Rename_ParsedPlaylistVideos : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "ParsedPlaylistVideos"); + + migrationBuilder.CreateTable( + name: "ParsedPlaylistItems", + columns: table => new + { + CreateDate = table.Column(nullable: false), + UpdateDate = table.Column(nullable: false), + Id = table.Column(nullable: false) + .Annotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn), + VideoId = table.Column(nullable: true), + IsProcessed = table.Column(nullable: false), + PlaylistId = table.Column(nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_ParsedPlaylistItems", x => x.Id); + table.ForeignKey( + name: "FK_ParsedPlaylistItems_Playlists_PlaylistId", + column: x => x.PlaylistId, + principalTable: "Playlists", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "IX_ParsedPlaylistItems_PlaylistId", + table: "ParsedPlaylistItems", + column: "PlaylistId"); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "ParsedPlaylistItems"); + + migrationBuilder.CreateTable( + name: "ParsedPlaylistVideos", + columns: table => new + { + Id = table.Column(nullable: false) + .Annotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn), + CreateDate = table.Column(nullable: false), + IsProcessed = table.Column(nullable: false), + PlaylistId = table.Column(nullable: false), + UpdateDate = table.Column(nullable: false), + VideoId = table.Column(nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_ParsedPlaylistVideos", x => x.Id); + table.ForeignKey( + name: "FK_ParsedPlaylistVideos_Playlists_PlaylistId", + column: x => x.PlaylistId, + principalTable: "Playlists", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "IX_ParsedPlaylistVideos_PlaylistId", + table: "ParsedPlaylistVideos", + column: "PlaylistId"); + } + } +} diff --git a/server/Migrations/20180507162159_AddVideoTypeToPlaylistItem.Designer.cs b/server/Migrations/20180507162159_AddVideoTypeToPlaylistItem.Designer.cs new file mode 100644 index 0000000..61bfb4e --- /dev/null +++ b/server/Migrations/20180507162159_AddVideoTypeToPlaylistItem.Designer.cs @@ -0,0 +1,392 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Migrations; +using PodNoms.Api.Persistence; + +namespace PodNoms.Api.Migrations +{ + [DbContext(typeof(PodnomsDbContext))] + [Migration("20180507162159_AddVideoTypeToPlaylistItem")] + partial class AddVideoTypeToPlaylistItem + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "2.1.0-preview2-30571") + .HasAnnotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRole", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken(); + + b.Property("Name") + .HasMaxLength(256); + + b.Property("NormalizedName") + .HasMaxLength(256); + + b.HasKey("Id"); + + b.HasIndex("NormalizedName") + .IsUnique() + .HasName("RoleNameIndex") + .HasFilter("[NormalizedName] IS NOT NULL"); + + b.ToTable("AspNetRoles"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("ClaimType"); + + b.Property("ClaimValue"); + + b.Property("RoleId") + .IsRequired(); + + b.HasKey("Id"); + + b.HasIndex("RoleId"); + + b.ToTable("AspNetRoleClaims"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("ClaimType"); + + b.Property("ClaimValue"); + + b.Property("UserId") + .IsRequired(); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.ToTable("AspNetUserClaims"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.Property("LoginProvider"); + + b.Property("ProviderKey"); + + b.Property("ProviderDisplayName"); + + b.Property("UserId") + .IsRequired(); + + b.HasKey("LoginProvider", "ProviderKey"); + + b.HasIndex("UserId"); + + b.ToTable("AspNetUserLogins"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole", b => + { + b.Property("UserId"); + + b.Property("RoleId"); + + b.HasKey("UserId", "RoleId"); + + b.HasIndex("RoleId"); + + b.ToTable("AspNetUserRoles"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.Property("UserId"); + + b.Property("LoginProvider"); + + b.Property("Name"); + + b.Property("Value"); + + b.HasKey("UserId", "LoginProvider", "Name"); + + b.ToTable("AspNetUserTokens"); + }); + + modelBuilder.Entity("PodNoms.Api.Models.ParsedPlaylistItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("CreateDate"); + + b.Property("IsProcessed"); + + b.Property("PlaylistId"); + + b.Property("UpdateDate"); + + b.Property("VideoId"); + + b.Property("VideoType"); + + b.HasKey("Id"); + + b.HasIndex("PlaylistId"); + + b.ToTable("ParsedPlaylistItems"); + }); + + modelBuilder.Entity("PodNoms.Api.Models.Playlist", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("CreateDate"); + + b.Property("PodcastId"); + + b.Property("SourceUrl"); + + b.Property("UpdateDate"); + + b.HasKey("Id"); + + b.HasIndex("PodcastId"); + + b.ToTable("Playlists"); + }); + + modelBuilder.Entity("PodNoms.Api.Models.Podcast", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("AppUserId"); + + b.Property("CreateDate"); + + b.Property("Description"); + + b.Property("Slug"); + + b.Property("TemporaryImageUrl"); + + b.Property("Title"); + + b.Property("Uid"); + + b.Property("UpdateDate"); + + b.HasKey("Id"); + + b.HasIndex("AppUserId"); + + b.ToTable("Podcasts"); + }); + + modelBuilder.Entity("PodNoms.Api.Models.PodcastEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("AudioFileSize"); + + b.Property("AudioLength"); + + b.Property("AudioUrl"); + + b.Property("Author"); + + b.Property("CreateDate"); + + b.Property("Description"); + + b.Property("ImageUrl"); + + b.Property("PlaylistId"); + + b.Property("PodcastId"); + + b.Property("Processed"); + + b.Property("ProcessingPayload"); + + b.Property("ProcessingStatus"); + + b.Property("SourceUrl"); + + b.Property("Title"); + + b.Property("Uid"); + + b.Property("UpdateDate"); + + b.HasKey("Id"); + + b.HasIndex("PlaylistId"); + + b.HasIndex("PodcastId"); + + b.ToTable("PodcastEntries"); + }); + + modelBuilder.Entity("PodNoms.Api.Services.Auth.ApplicationUser", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("AccessFailedCount"); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken(); + + b.Property("Email") + .HasMaxLength(256); + + b.Property("EmailConfirmed"); + + b.Property("FacebookId"); + + b.Property("FirstName"); + + b.Property("LastName"); + + b.Property("LockoutEnabled"); + + b.Property("LockoutEnd"); + + b.Property("NormalizedEmail") + .HasMaxLength(256); + + b.Property("NormalizedUserName") + .HasMaxLength(256); + + b.Property("PasswordHash"); + + b.Property("PhoneNumber"); + + b.Property("PhoneNumberConfirmed"); + + b.Property("PictureUrl"); + + b.Property("SecurityStamp"); + + b.Property("Slug"); + + b.Property("TwoFactorEnabled"); + + b.Property("UserName") + .HasMaxLength(256); + + b.HasKey("Id"); + + b.HasIndex("NormalizedEmail") + .HasName("EmailIndex"); + + b.HasIndex("NormalizedUserName") + .IsUnique() + .HasName("UserNameIndex") + .HasFilter("[NormalizedUserName] IS NOT NULL"); + + b.ToTable("AspNetUsers"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole") + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.HasOne("PodNoms.Api.Services.Auth.ApplicationUser") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.HasOne("PodNoms.Api.Services.Auth.ApplicationUser") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole") + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("PodNoms.Api.Services.Auth.ApplicationUser") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.HasOne("PodNoms.Api.Services.Auth.ApplicationUser") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("PodNoms.Api.Models.ParsedPlaylistItem", b => + { + b.HasOne("PodNoms.Api.Models.Playlist", "Playlist") + .WithMany("ParsedPlaylistItems") + .HasForeignKey("PlaylistId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("PodNoms.Api.Models.Playlist", b => + { + b.HasOne("PodNoms.Api.Models.Podcast", "Podcast") + .WithMany() + .HasForeignKey("PodcastId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("PodNoms.Api.Models.Podcast", b => + { + b.HasOne("PodNoms.Api.Services.Auth.ApplicationUser", "AppUser") + .WithMany() + .HasForeignKey("AppUserId"); + }); + + modelBuilder.Entity("PodNoms.Api.Models.PodcastEntry", b => + { + b.HasOne("PodNoms.Api.Models.Playlist") + .WithMany("PodcastEntries") + .HasForeignKey("PlaylistId"); + + b.HasOne("PodNoms.Api.Models.Podcast", "Podcast") + .WithMany("PodcastEntries") + .HasForeignKey("PodcastId") + .OnDelete(DeleteBehavior.Cascade); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/server/Migrations/20180507162159_AddVideoTypeToPlaylistItem.cs b/server/Migrations/20180507162159_AddVideoTypeToPlaylistItem.cs new file mode 100644 index 0000000..699b0a5 --- /dev/null +++ b/server/Migrations/20180507162159_AddVideoTypeToPlaylistItem.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace PodNoms.Api.Migrations +{ + public partial class AddVideoTypeToPlaylistItem : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "VideoType", + table: "ParsedPlaylistItems", + nullable: true); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "VideoType", + table: "ParsedPlaylistItems"); + } + } +} diff --git a/server/Migrations/PodnomsDbContextModelSnapshot.cs b/server/Migrations/PodnomsDbContextModelSnapshot.cs index 3bd4e5b..5b49344 100644 --- a/server/Migrations/PodnomsDbContextModelSnapshot.cs +++ b/server/Migrations/PodnomsDbContextModelSnapshot.cs @@ -127,6 +127,30 @@ namespace PodNoms.Api.Migrations b.ToTable("AspNetUserTokens"); }); + modelBuilder.Entity("PodNoms.Api.Models.ParsedPlaylistItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("CreateDate"); + + b.Property("IsProcessed"); + + b.Property("PlaylistId"); + + b.Property("UpdateDate"); + + b.Property("VideoId"); + + b.Property("VideoType"); + + b.HasKey("Id"); + + b.HasIndex("PlaylistId"); + + b.ToTable("ParsedPlaylistItems"); + }); + modelBuilder.Entity("PodNoms.Api.Models.Playlist", b => { b.Property("Id") @@ -154,14 +178,11 @@ namespace PodNoms.Api.Migrations b.Property("AppUserId"); - b.Property("CreateDate") - .ValueGeneratedOnAdd() - .HasDefaultValueSql("getdate()"); + b.Property("CreateDate"); b.Property("Description"); - b.Property("Slug") - .IsUnicode(true); + b.Property("Slug"); b.Property("TemporaryImageUrl"); @@ -191,9 +212,7 @@ namespace PodNoms.Api.Migrations b.Property("Author"); - b.Property("CreateDate") - .ValueGeneratedOnAdd() - .HasDefaultValueSql("getdate()"); + b.Property("CreateDate"); b.Property("Description"); @@ -332,6 +351,14 @@ namespace PodNoms.Api.Migrations .OnDelete(DeleteBehavior.Cascade); }); + modelBuilder.Entity("PodNoms.Api.Models.ParsedPlaylistItem", b => + { + b.HasOne("PodNoms.Api.Models.Playlist", "Playlist") + .WithMany("ParsedPlaylistItems") + .HasForeignKey("PlaylistId") + .OnDelete(DeleteBehavior.Cascade); + }); + modelBuilder.Entity("PodNoms.Api.Models.Playlist", b => { b.HasOne("PodNoms.Api.Models.Podcast", "Podcast") diff --git a/server/Models/AppSettings.cs b/server/Models/AppSettings.cs index e2fb106..32d062a 100644 --- a/server/Models/AppSettings.cs +++ b/server/Models/AppSettings.cs @@ -3,5 +3,6 @@ namespace PodNoms.Api.Models { public string Version { get; set; } public string SiteUrl { get; set; } public string RssUrl { get; set; } + public string GoogleApiKey { get; set; } } } \ No newline at end of file diff --git a/server/Models/Playlist.cs b/server/Models/Playlist.cs index faa81f1..0e2eee0 100644 --- a/server/Models/Playlist.cs +++ b/server/Models/Playlist.cs @@ -7,5 +7,17 @@ namespace PodNoms.Api.Models { public string SourceUrl { get; set; } public Podcast Podcast { get; set; } public List PodcastEntries { get; set; } + public List ParsedPlaylistItems { get; set; } + public Playlist() { + ParsedPlaylistItems = new List(); + } + } + public class ParsedPlaylistItem : BaseModel { + public int Id { get; set; } + public string VideoId { get; set; } + public string VideoType { get; set; } + public bool IsProcessed { get; set; } + public int PlaylistId { get; set; } + public Playlist Playlist { get; set; } } } \ No newline at end of file diff --git a/server/Persistence/IPlaylistRepository.cs b/server/Persistence/IPlaylistRepository.cs index 5261083..86a585a 100644 --- a/server/Persistence/IPlaylistRepository.cs +++ b/server/Persistence/IPlaylistRepository.cs @@ -7,5 +7,7 @@ namespace PodNoms.Api.Persistence { Task GetAsync(int id); Task> GetAllAsync(); Task AddOrUpdateAsync(Playlist playlist); + Task GetParsedItem(string itemId, int playlistId); + Task> GetUnprocessedItems(); } } \ No newline at end of file diff --git a/server/Persistence/PlaylistRepository.cs b/server/Persistence/PlaylistRepository.cs index 12d0179..1a9a2d7 100644 --- a/server/Persistence/PlaylistRepository.cs +++ b/server/Persistence/PlaylistRepository.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using PodNoms.Api.Models; @@ -18,7 +19,7 @@ namespace PodNoms.Api.Persistence { return entry; } public async Task> GetAllAsync() { - return await _context.Playlists.ToListAsync(); + return await _context.Playlists.Include(p => p.ParsedPlaylistItems).ToListAsync(); } public async Task AddOrUpdateAsync(Playlist playlist) { if (playlist.Id != 0) { @@ -30,5 +31,20 @@ namespace PodNoms.Api.Persistence { } return playlist; } + public async Task GetParsedItem(string itemId, int playlistId) { + return await _context.ParsedPlaylistItems + .Include(i => i.Playlist) + .Include(i => i.Playlist.Podcast) + .Include(i => i.Playlist.Podcast.AppUser) + .SingleOrDefaultAsync(i => i.VideoId == itemId && i.PlaylistId == playlistId); + } + public async Task> GetUnprocessedItems() { + return await _context.ParsedPlaylistItems + .Where(p => p.IsProcessed == false) + .Include(i => i.Playlist) + .Include(i => i.Playlist.Podcast) + .Include(i => i.Playlist.Podcast.AppUser) + .ToListAsync(); + } } } \ No newline at end of file diff --git a/server/Persistence/PodnomsContext.cs b/server/Persistence/PodnomsContext.cs index 8baf0b6..76bdc14 100644 --- a/server/Persistence/PodnomsContext.cs +++ b/server/Persistence/PodnomsContext.cs @@ -1,5 +1,8 @@ using System; using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Microsoft.AspNetCore.Identity.EntityFrameworkCore; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Design; @@ -15,24 +18,37 @@ namespace PodNoms.Api.Persistence { protected override void OnModelCreating(ModelBuilder modelBuilder) { base.OnModelCreating(modelBuilder); - - - modelBuilder.Entity() - .Property(b => b.CreateDate) - .HasDefaultValueSql("getdate()"); - - modelBuilder.Entity() - .Property(b => b.Slug) - .IsUnicode(true); - - modelBuilder.Entity() - .Property(b => b.CreateDate) - .HasDefaultValueSql("getdate()"); - } + public override int SaveChanges() { + _addTimestamps(); + return base.SaveChanges(); + } + public override Task SaveChangesAsync(bool acceptAllChangesOnSuccess, CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { + _addTimestamps(); + return base.SaveChangesAsync(acceptAllChangesOnSuccess, cancellationToken); + } + public override Task SaveChangesAsync(CancellationToken cancellationToken = default) { + _addTimestamps(); + return base.SaveChangesAsync(cancellationToken); + } + void _addTimestamps() { + var modifiedEntries = ChangeTracker.Entries() + .Where(x => (x.State == EntityState.Added || x.State == EntityState.Modified)); + var now = DateTime.Now; + foreach (var entry in modifiedEntries) { + var entity = entry.Entity as BaseModel; + if (entity != null) { + if (entry.State == EntityState.Added) { + entity.CreateDate = now; + } + entity.UpdateDate = now; + } + } + } public DbSet Podcasts { get; set; } public DbSet PodcastEntries { get; set; } public DbSet Playlists { get; set; } + public DbSet ParsedPlaylistItems { get; set; } } } \ No newline at end of file diff --git a/server/PodNoms.Api.csproj b/server/PodNoms.Api.csproj index 476f834..5e23e71 100644 --- a/server/PodNoms.Api.csproj +++ b/server/PodNoms.Api.csproj @@ -3,6 +3,7 @@ netcoreapp2.1 aspnet-PodNoms.Api-1E27B6DE-BA4B-4F75-BBF8-CA34FB4D260A latest + PodNoms.Api @@ -13,10 +14,8 @@ - - - - + + diff --git a/server/Services/Downloader/AudioDownloader.cs b/server/Services/Downloader/AudioDownloader.cs index 1e99c00..f68d22d 100644 --- a/server/Services/Downloader/AudioDownloader.cs +++ b/server/Services/Downloader/AudioDownloader.cs @@ -54,24 +54,22 @@ namespace PodNoms.Api.Services.Downloader { return $"{{\"Error\": \"{ex.Message}\"}}"; } } - public async Task GetInfo() { + public AudioType GetInfo() { var ret = AudioType.Invalid; - await Task.Run(() => { - var youtubeDl = new YoutubeDL(); - youtubeDl.VideoUrl = this._url; - var info = youtubeDl.GetDownloadInfo(); + var yt = new YoutubeDL(); + yt.VideoUrl = this._url; + var info = yt.GetDownloadInfo(); - if (info != null && - (info.Errors.Count == 0 || info.VideoSize != null)) { - this._properties = info; - // have to dump playlist handling for now - if (info is PlaylistDownloadInfo && ((PlaylistDownloadInfo)info).Videos.Count > 0) { - ret = AudioType.Playlist; - } else if (info is VideoDownloadInfo) { - ret = AudioType.Valid; - } + if (info != null && + (info.Errors.Count == 0 || info.VideoSize != null)) { + this._properties = info; + // have to dump playlist handling for now + if (info is PlaylistDownloadInfo && ((PlaylistDownloadInfo)info).Videos.Count > 0) { + ret = AudioType.Playlist; + } else if (info is VideoDownloadInfo) { + ret = AudioType.Valid; } - }); + } return ret; } @@ -89,10 +87,12 @@ namespace PodNoms.Api.Services.Downloader { yt.StandardOutputEvent += (sender, output) => { if (output.Contains("%")) { var progress = _parseProgress(output); + Console.WriteLine($"Processing {progress.CurrentSpeed} {progress.ETA} {progress.Percentage}"); if (DownloadProgress != null) { DownloadProgress(this, progress); } } else { + Console.WriteLine(output); if (PostProcessing != null) { PostProcessing(this, output); } diff --git a/server/Services/Jobs/JobBootstrapper.cs b/server/Services/Jobs/JobBootstrapper.cs index 577002d..083559b 100644 --- a/server/Services/Jobs/JobBootstrapper.cs +++ b/server/Services/Jobs/JobBootstrapper.cs @@ -10,7 +10,6 @@ namespace PodNoms.Api.Services.Jobs { public static void BootstrapJobs() { RecurringJob.AddOrUpdate(x => x.Execute(), Cron.Daily(1)); RecurringJob.AddOrUpdate(x => x.Execute(), Cron.Daily(1, 30)); - // BackgroundJob.Schedule(x => x.Execute(), TimeSpan.FromSeconds(1)); } } diff --git a/server/Services/Jobs/ProcessPlaylistItemJob.cs b/server/Services/Jobs/ProcessPlaylistItemJob.cs new file mode 100644 index 0000000..b211af1 --- /dev/null +++ b/server/Services/Jobs/ProcessPlaylistItemJob.cs @@ -0,0 +1,80 @@ +using System.Threading.Tasks; +using Hangfire; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using PodNoms.Api.Models; +using PodNoms.Api.Persistence; +using PodNoms.Api.Services.Downloader; +using PodNoms.Api.Services.Processor; +using PodNoms.Api.Utils.RemoteParsers; + +namespace PodNoms.Api.Services.Jobs { + public class ProcessPlaylistItemJob : IJob { + public readonly IPlaylistRepository _playlistRepository; + public readonly IEntryRepository _entryRepository; + private readonly IAudioUploadProcessService _uploadService; + private readonly IConfiguration _options; + private readonly IPodcastRepository _podcastRepository; + private readonly ApplicationsSettings _applicationsSettings; + private readonly ILogger _logger; + private readonly IUnitOfWork _unitOfWork; + public ProcessPlaylistItemJob(IPlaylistRepository playlistRepository, IEntryRepository entryRepository, + IAudioUploadProcessService uploadService, IConfiguration options, + IPodcastRepository podcastRepository, IOptions applicationsSettings, + IUnitOfWork unitOfWork, ILogger logger) { + this._unitOfWork = unitOfWork; + this._playlistRepository = playlistRepository; + this._entryRepository = entryRepository; + this._uploadService = uploadService; + this._options = options; + this._podcastRepository = podcastRepository; + this._applicationsSettings = applicationsSettings.Value; + this._logger = logger; + } + public async Task Execute() { + var items = await _playlistRepository.GetUnprocessedItems(); + foreach (var item in items) { + await ExecuteForItem(item.VideoId, item.Playlist.Id); + } + } + public async Task ExecuteForItem(string itemId, int playlistId) { + var item = await _playlistRepository.GetParsedItem(itemId, playlistId); + if (item != null && !string.IsNullOrEmpty(item.VideoType) && item.VideoType.Equals("youtube")) { + var url = $"https://www.youtube.com/watch?v={item.VideoId}"; + var downloader = new AudioDownloader(url, _applicationsSettings.Downloader); + var info = downloader.GetInfo(); + if (info == AudioType.Valid) { + var podcast = await _podcastRepository.GetAsync(item.Playlist.PodcastId); + var uid = System.Guid.NewGuid().ToString(); + var file = downloader.DownloadAudio(uid); + if (System.IO.File.Exists(file)) { + //we have the file so lets create the entry and ship to CDN + var entry = new PodcastEntry { + Title = downloader.Properties?.Title, + Uid = uid, + Description = downloader.Properties?.Description, + ProcessingStatus = ProcessingStatus.Uploading, + ImageUrl = downloader.Properties?.Thumbnail + }; + podcast.PodcastEntries.Add(entry); + await _unitOfWork.CompleteAsync(); + var uploaded = await _uploadService.UploadAudio(entry.Id, file); + if (uploaded) { + item.IsProcessed = true; + await _unitOfWork.CompleteAsync(); + BackgroundJob.Enqueue( + service => service.NotifyUser(entry.Podcast.AppUser.Id, "PodNoms", $"{entry.Title} has finished processing", + entry.Podcast.GetThumbnailUrl( + this._options.GetSection("Storage")["CdnUrl"], + this._options.GetSection("ImageFileStorageSettings")["ContainerName"]) + )); + } + } + } else { + _logger.LogError($"Processing playlist item {itemId} failed"); + } + } + } + } +} \ No newline at end of file diff --git a/server/Services/Jobs/ProcessPlaylistsJob.cs b/server/Services/Jobs/ProcessPlaylistsJob.cs index 64dfb26..280f87f 100644 --- a/server/Services/Jobs/ProcessPlaylistsJob.cs +++ b/server/Services/Jobs/ProcessPlaylistsJob.cs @@ -1,13 +1,16 @@ +using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; +using Hangfire; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using NYoutubeDL.Models; using PodNoms.Api.Models; using PodNoms.Api.Persistence; using PodNoms.Api.Services.Downloader; +using PodNoms.Api.Utils.RemoteParsers; using static NYoutubeDL.Helpers.Enums; namespace PodNoms.Api.Services.Jobs { @@ -16,44 +19,51 @@ namespace PodNoms.Api.Services.Jobs { public readonly IEntryRepository _entryRepository; private readonly ApplicationsSettings _applicationsSettings; private readonly ILogger _logger; + private readonly YouTubeParser _youTubeParser; + private readonly MixcloudParser _mixcloudParser; + private readonly IUnitOfWork _unitOfWork; - public ProcessPlaylistsJob(IPlaylistRepository playlistRepository, - IEntryRepository entryRepository, IOptions applicationsSettings, ILoggerFactory logger) { - _playlistRepository = playlistRepository; - _entryRepository = entryRepository; - _applicationsSettings = applicationsSettings.Value; - _logger = logger.CreateLogger(); + public ProcessPlaylistsJob(IPlaylistRepository playlistRepository, IEntryRepository entryRepository, + IUnitOfWork unitOfWork, IOptions applicationsSettings, + ILoggerFactory logger, YouTubeParser youTubeParser, MixcloudParser mixcloudParser) { + this._unitOfWork = unitOfWork; + this._youTubeParser = youTubeParser; + this._mixcloudParser = mixcloudParser; + this._playlistRepository = playlistRepository; + this._entryRepository = entryRepository; + this._applicationsSettings = applicationsSettings.Value; + this._logger = logger.CreateLogger(); } public async Task Execute() { - var playists = await _playlistRepository.GetAllAsync(); + var playlists = await _playlistRepository.GetAllAsync(); + var resultList = new List(); - foreach (var playlist in playists) { + foreach (var playlist in playlists) { var downloader = new AudioDownloader(playlist.SourceUrl, _applicationsSettings.Downloader); - var info = await downloader.GetInfo(); + var info = downloader.GetInfo(); + var id = ((PlaylistDownloadInfo)downloader.RawProperties).Id; if (info == AudioType.Playlist && downloader.RawProperties is PlaylistDownloadInfo) { - var list = ((PlaylistDownloadInfo)downloader.RawProperties).Videos - .OrderByDescending(x => x.Id) - .Take(10); - StringBuilder br = new StringBuilder(); - foreach (var item in list) { - _logger.LogDebug($"Processing: {item.Id} - {item.Url}"); - br.Append($"Processing: {item.Id} - {item.Title}\n"); - - var outputDir = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); - Directory.CreateDirectory(outputDir); - - var yt = new NYoutubeDL.YoutubeDL(playlist.SourceUrl); - yt.Options.PostProcessingOptions.ExtractAudio = true; - yt.Options.PostProcessingOptions.AudioFormat = AudioFormat.mp3; - yt.Options.VideoSelectionOptions.PlaylistItems = "1,2,3"; - yt.Options.FilesystemOptions.Output = Path.Combine(outputDir, "%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s"); - - var p = yt.Download(); - p.WaitForExit(); + if (_youTubeParser.ValidateUrl(playlist.SourceUrl)) { + var searchTerm = (playlist.SourceUrl.Contains("/user/")) ? "forUsername" : "id"; + resultList = await _youTubeParser.GetPlaylistEntriesForId(id); + //make sure the items are sorted in ascending date order + //so they will be processed in the order they were created + } else if (_mixcloudParser.ValidateUrl(playlist.SourceUrl)) { + resultList = await _mixcloudParser.GetEntries(id); } - _logger.LogDebug(br.ToString()); + } + foreach (var item in resultList?.OrderBy(r => r.UploadDate)) { + if (!playlist.ParsedPlaylistItems.Any(p => p.VideoId == item.Id)) { + playlist.ParsedPlaylistItems.Add(new ParsedPlaylistItem { + VideoId = item.Id, + VideoType = item.VideoType + }); + BackgroundJob.Enqueue(service => service.ExecuteForItem(item.Id, playlist.Id)); + } + } + await _unitOfWork.CompleteAsync(); } } } diff --git a/server/Services/NYT/Helpers/NotifyPropertyChangedEx.cs b/server/Services/NYT/Helpers/NotifyPropertyChangedEx.cs index 91e69b4..186b141 100644 --- a/server/Services/NYT/Helpers/NotifyPropertyChangedEx.cs +++ b/server/Services/NYT/Helpers/NotifyPropertyChangedEx.cs @@ -1,15 +1,15 @@ // Copyright 2017 Brian Allred -// +// // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to // deal in the Software without restriction, including without limitation the // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or // sell copies of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: -// +// // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// +// // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE @@ -28,7 +28,7 @@ namespace NYoutubeDL.Helpers #endregion - + /// /// Abstract class that extends the functionality of a traditional NotifyPropertyChanged implementation /// diff --git a/server/Services/NYT/Models/DownloadInfo.cs b/server/Services/NYT/Models/DownloadInfo.cs index 64264aa..aee8ccd 100644 --- a/server/Services/NYT/Models/DownloadInfo.cs +++ b/server/Services/NYT/Models/DownloadInfo.cs @@ -1,15 +1,15 @@ // Copyright 2017 Brian Allred -// +// // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to // deal in the Software without restriction, including without limitation the // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or // sell copies of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: -// +// // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// +// // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE @@ -18,8 +18,7 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -namespace NYoutubeDL.Models -{ +namespace NYoutubeDL.Models { #region Using using System; @@ -32,8 +31,7 @@ namespace NYoutubeDL.Models /// /// Class holding data about the current download, which is parsed from youtube-dl's standard output /// - public class DownloadInfo : NotifyPropertyChangedEx - { + public class DownloadInfo : NotifyPropertyChangedEx { protected const string ALREADY = "already"; protected const string DOWNLOADRATESTRING = "iB/s"; @@ -51,7 +49,7 @@ namespace NYoutubeDL.Models private string eta; private string status = Enums.DownloadStatus.WAITING.ToString(); - + private string id; private string title; private int videoProgress; @@ -61,8 +59,7 @@ namespace NYoutubeDL.Models /// /// The current download rate /// - public string DownloadRate - { + public string DownloadRate { get => this.downloadRate; set => this.SetField(ref this.downloadRate, value); } @@ -75,8 +72,7 @@ namespace NYoutubeDL.Models /// /// The current download's estimated time remaining /// - public string Eta - { + public string Eta { get => this.eta; set => this.SetField(ref this.eta, value); } @@ -84,29 +80,30 @@ namespace NYoutubeDL.Models /// /// The status of the video currently downloading /// - public string Status - { + public string Status { get => this.status; - set - { + set { if (!this.status.Equals(Enums.DownloadStatus.ERROR.ToString()) && - !this.status.Equals(Enums.DownloadStatus.WARNING.ToString())) - { + !this.status.Equals(Enums.DownloadStatus.WARNING.ToString())) { this.SetField(ref this.status, value); - } - else if (value.Equals(Enums.DownloadStatus.ERROR.ToString()) && - this.status.Equals(Enums.DownloadStatus.WARNING.ToString())) - { + } else if (value.Equals(Enums.DownloadStatus.ERROR.ToString()) && + this.status.Equals(Enums.DownloadStatus.WARNING.ToString())) { this.SetField(ref this.status, value); } } } + /// + /// The id of the playlist + /// + public string Id { + get => this.id; + set => this.SetField(ref this.id, value); + } /// /// The title of the video currently downloading /// - public string Title - { + public string Title { get => this.title; set => this.SetField(ref this.title, value); } @@ -114,23 +111,16 @@ namespace NYoutubeDL.Models /// /// The current download progresss /// - public int VideoProgress - { + public int VideoProgress { get => this.videoProgress; - set - { + set { this.SetField(ref this.videoProgress, value); - if (value == 0) - { + if (value == 0) { this.Status = Enums.DownloadStatus.WAITING.ToString(); - } - else if (value == 100) - { + } else if (value == 100) { this.Status = Enums.DownloadStatus.DONE.ToString(); - } - else - { + } else { this.Status = Enums.DownloadStatus.DOWNLOADING.ToString(); } } @@ -139,8 +129,7 @@ namespace NYoutubeDL.Models /// /// The current download's total size /// - public string VideoSize - { + public string VideoSize { get => this.videoSize; set => this.SetField(ref this.videoSize, value); } @@ -150,31 +139,25 @@ namespace NYoutubeDL.Models /// public List Warnings { get; } = new List(); - internal static DownloadInfo CreateDownloadInfo(string output) - { - try - { + internal static DownloadInfo CreateDownloadInfo(string output) { + if (string.IsNullOrEmpty(output) || output.Equals("null")) + return null; + + try { PlaylistInfo info = JsonConvert.DeserializeObject(output); - if (!string.IsNullOrEmpty(info._type) && info._type.Equals("playlist")) - { + if (!string.IsNullOrEmpty(info._type) && info._type.Equals("playlist")) { return new PlaylistDownloadInfo(info); } - } - catch (Exception ex) - { + } catch (Exception ex) { Console.WriteLine(ex); } - try - { + try { VideoInfo info = JsonConvert.DeserializeObject(output); - if (!string.IsNullOrEmpty(info.title)) - { + if (!string.IsNullOrEmpty(info.title)) { return new VideoDownloadInfo(info); } - } - catch (Exception ex) - { + } catch (Exception ex) { Console.WriteLine(ex); } @@ -186,57 +169,45 @@ namespace NYoutubeDL.Models /// public event EventHandler ErrorEvent; - internal virtual void ParseError(object sender, string error) - { + internal virtual void ParseError(object sender, string error) { this.ErrorEvent?.Invoke(this, error); - if (error.Contains("WARNING")) - { + if (error.Contains("WARNING")) { this.Warnings.Add(error); this.Status = Enums.DownloadStatus.WARNING.ToString(); - } - else if (error.Contains("ERROR")) - { + } else if (error.Contains("ERROR")) { this.Errors.Add(error); this.Status = Enums.DownloadStatus.ERROR.ToString(); } } - internal virtual void ParseOutput(object sender, string output) - { - try - { - if (output.Contains("%")) - { + internal virtual void ParseOutput(object sender, string output) { + try { + if (output.Contains("%")) { int progressIndex = output.LastIndexOf(' ', output.IndexOf('%')) + 1; string progressString = output.Substring(progressIndex, output.IndexOf('%') - progressIndex); - this.VideoProgress = (int) Math.Round(double.Parse(progressString)); + this.VideoProgress = (int)Math.Round(double.Parse(progressString)); int sizeIndex = output.LastIndexOf(' ', output.IndexOf(DOWNLOADSIZESTRING)) + 1; string sizeString = output.Substring(sizeIndex, output.IndexOf(DOWNLOADSIZESTRING) - sizeIndex + 2); this.VideoSize = sizeString; } - if (output.Contains(DOWNLOADRATESTRING)) - { + if (output.Contains(DOWNLOADRATESTRING)) { int rateIndex = output.LastIndexOf(' ', output.LastIndexOf(DOWNLOADRATESTRING)) + 1; string rateString = output.Substring(rateIndex, output.LastIndexOf(DOWNLOADRATESTRING) - rateIndex + 4); this.DownloadRate = rateString; } - if (output.Contains(ETASTRING)) - { + if (output.Contains(ETASTRING)) { this.Eta = output.Substring(output.LastIndexOf(' ') + 1); } - if (output.Contains(ALREADY)) - { + if (output.Contains(ALREADY)) { this.Status = Enums.DownloadStatus.DONE.ToString(); this.VideoProgress = 100; } - } - catch (Exception) - { + } catch (Exception) { } } } diff --git a/server/Services/NYT/Models/PlaylistDownloadInfo.cs b/server/Services/NYT/Models/PlaylistDownloadInfo.cs index fcc41c8..80388b7 100644 --- a/server/Services/NYT/Models/PlaylistDownloadInfo.cs +++ b/server/Services/NYT/Models/PlaylistDownloadInfo.cs @@ -1,15 +1,15 @@ // Copyright 2017 Brian Allred -// +// // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to // deal in the Software without restriction, including without limitation the // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or // sell copies of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: -// +// // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// +// // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE @@ -39,6 +39,7 @@ namespace NYoutubeDL.Models public PlaylistDownloadInfo(PlaylistInfo info) { + this.Id = info.id; this.Title = info.title; foreach (VideoInfo videoInfo in info.entries) { diff --git a/server/Services/NYT/Models/VideoDownloadInfo.cs b/server/Services/NYT/Models/VideoDownloadInfo.cs index 0e223a9..9eb9885 100644 --- a/server/Services/NYT/Models/VideoDownloadInfo.cs +++ b/server/Services/NYT/Models/VideoDownloadInfo.cs @@ -1,15 +1,15 @@ // Copyright 2017 Brian Allred -// +// // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to // deal in the Software without restriction, including without limitation the // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or // sell copies of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: -// +// // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// +// // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE @@ -180,11 +180,6 @@ namespace NYoutubeDL.Models public int? Width { get; } - /// - /// The ID string of the video - /// - public string Id { get; } - public List Thumbnails { get; } = new List(); public List Tags { get; } diff --git a/server/Services/Processor/UrlProcessService.cs b/server/Services/Processor/UrlProcessService.cs index a95d6c8..4db1ce7 100644 --- a/server/Services/Processor/UrlProcessService.cs +++ b/server/Services/Processor/UrlProcessService.cs @@ -57,7 +57,7 @@ namespace PodNoms.Api.Services.Processor { public async Task GetInformation(PodcastEntry entry) { var downloader = new AudioDownloader(entry.SourceUrl, _applicationsSettings.Downloader); - var ret = await downloader.GetInfo(); + var ret = downloader.GetInfo(); if (ret == AudioType.Valid) { entry.Title = downloader.Properties?.Title; entry.Description = downloader.Properties?.Description; diff --git a/server/Startup.cs b/server/Startup.cs index ad27d16..e8a1687 100644 --- a/server/Startup.cs +++ b/server/Startup.cs @@ -46,6 +46,7 @@ using Microsoft.AspNetCore.Identity; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.AspNetCore.Identity.UI.Services; +using PodNoms.Api.Utils.RemoteParsers; namespace PodNoms.Api { public class Startup { @@ -102,7 +103,7 @@ namespace PodNoms.Api { services.Configure(Configuration.GetSection("ImageFileStorageSettings")); services.Configure(Configuration.GetSection("AudioFileStorageSettings")); services.Configure(options => { - options.ValueCountLimit = 10; + // options.ValueCountLimit = 10; options.ValueLengthLimit = int.MaxValue; options.MemoryBufferThreshold = Int32.MaxValue; options.MultipartBodyLengthLimit = long.MaxValue; @@ -112,6 +113,10 @@ namespace PodNoms.Api { e.AddProfile(new MappingProvider(Configuration)); }); + services.AddHttpClient("mixcloud", c => { + c.BaseAddress = new Uri("https://api.mixcloud.com/"); + c.DefaultRequestHeaders.Add("Accept", "application/json"); + }); services.AddHttpClient(); var jwtAppSettingOptions = Configuration.GetSection(nameof(JwtIssuerOptions)); @@ -220,6 +225,8 @@ namespace PodNoms.Api { services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); + services.AddScoped(); services.AddHttpClient(); //register the codepages (required for slugify) diff --git a/server/Utils/RemoteParsers/MixcloudParser.cs b/server/Utils/RemoteParsers/MixcloudParser.cs new file mode 100644 index 0000000..260a540 --- /dev/null +++ b/server/Utils/RemoteParsers/MixcloudParser.cs @@ -0,0 +1,36 @@ +using System.Collections.Generic; +using System.Net.Http; +using System.Linq; +using System.Threading.Tasks; +using Newtonsoft.Json; +using System.Text.RegularExpressions; + +namespace PodNoms.Api.Utils.RemoteParsers { + public class MixcloudParser { + const string URL_REGEX = @"^(http(s)?:\/\/)?((w){3}.)?mixcloud?(\.com)?\/.+"; + + private readonly IHttpClientFactory _httpClientFactory; + + public MixcloudParser(IHttpClientFactory httpClientFactory) { + this._httpClientFactory = httpClientFactory; + } + public bool ValidateUrl(string url) { + var regex = new Regex(URL_REGEX); + var result = regex.Match(url); + return result.Success; + } + public async Task> GetEntries(string identifier) { + var client = _httpClientFactory.CreateClient("mixcloud"); + var result = await client.GetAsync(identifier); + if (result.IsSuccessStatusCode) { + var typed = JsonConvert.DeserializeObject(await result.Content.ReadAsStringAsync()); + return typed.data[0].cloudcasts.Select(c => new ParsedItemResult { + Id = c.key, + VideoType = "mixcloud", + UploadDate = c.updated_time + }).ToList(); + } + return null; + } + } +} \ No newline at end of file diff --git a/server/Utils/RemoteParsers/MixcloudViewModels.cs b/server/Utils/RemoteParsers/MixcloudViewModels.cs new file mode 100644 index 0000000..bef22a1 --- /dev/null +++ b/server/Utils/RemoteParsers/MixcloudViewModels.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Generic; + +namespace PodNoms.Api.Utils.RemoteParsers { + + + public class Paging { + public string previous { get; set; } + public string next { get; set; } + } + + public class Pictures { + public string medium { get; set; } + public string extra_large { get; set; } + public string large { get; set; } + public string medium_mobile { get; set; } + public string small { get; set; } + public string thumbnail { get; set; } + } + + public class From { + public string url { get; set; } + public string username { get; set; } + public string name { get; set; } + public string key { get; set; } + public Pictures pictures { get; set; } + } + + public class Tag { + public string url { get; set; } + public string name { get; set; } + public string key { get; set; } + } + + public class User { + public string url { get; set; } + public string username { get; set; } + public string name { get; set; } + public string key { get; set; } + } + + public class Cloudcast { + public IList tags { get; set; } + public int play_count { get; set; } + public User user { get; set; } + public string key { get; set; } + public DateTime created_time { get; set; } + public int audio_length { get; set; } + public string slug { get; set; } + public int favorite_count { get; set; } + public int listener_count { get; set; } + public string name { get; set; } + public string url { get; set; } + public int repost_count { get; set; } + public DateTime updated_time { get; set; } + public int comment_count { get; set; } + } + + public class Datum { + public From from { get; set; } + public string title { get; set; } + public string url { get; set; } + public string key { get; set; } + public DateTime created_time { get; set; } + public IList cloudcasts { get; set; } + public string type { get; set; } + } + + public class MixcloudResult { + public Paging paging { get; set; } + public IList data { get; set; } + public string name { get; set; } + } + +} \ No newline at end of file diff --git a/server/Utils/RemoteParsers/ParsedItemResult.cs b/server/Utils/RemoteParsers/ParsedItemResult.cs new file mode 100644 index 0000000..9cb0877 --- /dev/null +++ b/server/Utils/RemoteParsers/ParsedItemResult.cs @@ -0,0 +1,9 @@ +using System; + +namespace PodNoms.Api.Utils.RemoteParsers { + public class ParsedItemResult { + public string Id { get; set; } + public string VideoType { get; set; } + public DateTime? UploadDate { get; set; } + } +} \ No newline at end of file diff --git a/server/Utils/RemoteParsers/YouTubeParser.cs b/server/Utils/RemoteParsers/YouTubeParser.cs new file mode 100644 index 0000000..f53367e --- /dev/null +++ b/server/Utils/RemoteParsers/YouTubeParser.cs @@ -0,0 +1,61 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Google.Apis.Services; +using Google.Apis.YouTube.v3; +using Microsoft.Extensions.Options; +using PodNoms.Api.Models; + +namespace PodNoms.Api.Utils.RemoteParsers { + public partial class YouTubeParser { + const string URL_REGEX = @"^(http(s)?:\/\/)?((w){3}.)?youtu(be|.be)?(\.com)?\/.+"; + private readonly AppSettings _settings; + private YouTubeService youtube; + public YouTubeParser(IOptions options) { + this._settings = options.Value; + this.youtube = _getYouTubeService(); + } + private YouTubeService _getYouTubeService() { + return new YouTubeService(new BaseClientService.Initializer() { + ApiKey = _settings.GoogleApiKey, + ApplicationName = this.GetType().ToString() + }); + } + + public bool ValidateUrl(string url) { + var regex = new Regex(URL_REGEX); + var result = regex.Match(url); + return result.Success; + } + public async Task> GetPlaylistEntriesForId(string id, int nCount = 10) { + var playlistRequest = youtube.PlaylistItems.List("contentDetails"); + playlistRequest.PlaylistId = id; + playlistRequest.MaxResults = nCount; + var plists = await playlistRequest.ExecuteAsync(); + return plists.Items + .Select(p => new ParsedItemResult { + Id = p.ContentDetails.VideoId, + VideoType = "youtube", + UploadDate = p.ContentDetails.VideoPublishedAt + }).ToList(); + } + public async Task> GetPlaylistEntriesForChannelName(string channelName, string searchType, int nCount = 10) { + + var request = youtube.Channels.List("contentDetails"); + if (searchType.Equals("id")) + request.Id = channelName; + else + request.ForUsername = channelName; + request.MaxResults = 1; + var resp = await request.ExecuteAsync(); + if (resp.Items.Count == 1) { + var uploadListId = resp.Items[0].ContentDetails.RelatedPlaylists.Uploads; + if (!string.IsNullOrEmpty(uploadListId)) { + return await GetPlaylistEntriesForId(uploadListId, nCount); + } + } + return null; + } + } +} \ No newline at end of file diff --git a/server/appsettings.json b/server/appsettings.json index 51b79c8..5e24936 100644 --- a/server/appsettings.json +++ b/server/appsettings.json @@ -8,9 +8,7 @@ } }, "App": { - "Version": "0.23.0", - "SiteUrl": "http://localhost:4200", - "RssUrl": "http://localhost:5000/rss/" + "Version": "0.23.0" }, "ConnectionStrings": { "DefaultConnection": "server=localhost;database=PodNoms;user id=sa;password=cTXu1nJLCpC/c",