Merge branch 'feature/storage_reporting' into develop

This commit is contained in:
Fergal Moran
2018-05-13 03:04:07 +01:00
26 changed files with 265 additions and 88 deletions

View File

@@ -31,6 +31,8 @@
"node_modules/popper.js/dist/umd/popper.min.js",
"node_modules/bootstrap/dist/js/bootstrap.js",
"node_modules/quill/dist/quill.js",
"node_modules/easy-pie-chart/dist/easypiechart.js",
"node_modules/easy-pie-chart/dist/jquery.easypiechart.js",
"node_modules/howler/dist/howler.js"
]
},
@@ -73,32 +75,6 @@
"browserTarget": "podnoms-web:build"
}
},
"test": {
"builder": "@angular-devkit/build-angular:karma",
"options": {
"main": "src/test.ts",
"karmaConfig": "./karma.conf.js",
"polyfills": "src/polyfills.ts",
"tsConfig": "src/tsconfig.spec.json",
"scripts": [
"node_modules/jquery/dist/jquery.js",
"node_modules/cookieconsent/build/cookieconsent.min.js",
"node_modules/tether/dist/js/tether.js",
"node_modules/popper.js/dist/umd/popper.min.js",
"node_modules/bootstrap/dist/js/bootstrap.js",
"node_modules/quill/dist/quill.js",
"node_modules/howler/dist/howler.js"
],
"styles": ["src/styles.css"],
"assets": [
"src/assets",
"src/favicon.ico",
"src/firebase-messaging-sw.js",
"src/facebook-auth.html",
"src/manifest.json"
]
}
},
"lint": {
"builder": "@angular-devkit/build-angular:tslint",
"options": {
@@ -110,27 +86,6 @@
}
}
}
},
"podnoms-web-e2e": {
"root": "",
"sourceRoot": "",
"projectType": "application",
"architect": {
"e2e": {
"builder": "@angular-devkit/build-angular:protractor",
"options": {
"protractorConfig": "./protractor.conf.js",
"devServerTarget": "podnoms-web:serve"
}
},
"lint": {
"builder": "@angular-devkit/build-angular:tslint",
"options": {
"tsConfig": ["e2e/tsconfig.e2e.json"],
"exclude": []
}
}
}
}
},
"defaultProject": "podnoms-web",

View File

@@ -6,7 +6,6 @@
"ng": "ng",
"start": "ng serve --aot",
"build": "ng build",
"test": "ng test",
"lint": "ng lint"
},
"ngrxGen": {
@@ -29,6 +28,7 @@
"@ngrx/store": "^5.1.0",
"@ngrx/store-devtools": "^5.1.0",
"@qontu/ngx-inline-editor": "^0.2.0-alpha.12",
"@types/jquery": "^3.3.1",
"angular2-jwt": "^0.2.3",
"angular2-moment": "^1.8.0",
"angularfire2": "^5.0.0-rc.7",
@@ -38,12 +38,14 @@
"cookieconsent": "^3.0.6",
"core-js": "^2.5.3",
"dropzone": "^5.3.0",
"easy-pie-chart": "^2.1.7",
"firebase": "^4.12.1",
"font-awesome": "^4.7.0",
"howler": "^2.0.9",
"jquery": "^3.3.1",
"lodash": "^4.17.5",
"ng2-toasty": "^4.0.3",
"ng2modules-easypiechart": "0.0.4",
"ngx-bootstrap": "^2.0.4",
"ngx-clipboard": "^10.0.0",
"ngx-cookieconsent": "^1.0.1",
@@ -70,12 +72,6 @@
"codelyzer": "~4.2.1",
"jasmine-core": "~2.99.1",
"jasmine-spec-reporter": "~4.2.1",
"karma": "~1.7.1",
"karma-chrome-launcher": "~2.2.0",
"karma-coverage-istanbul-reporter": "~1.4.2",
"karma-jasmine": "~1.1.1",
"karma-jasmine-html-reporter": "^0.2.2",
"protractor": "~5.3.0",
"ts-node": "~5.0.1",
"tslint": "~5.9.1",
"@types/applicationinsights-js": "^1.0.5"

View File

@@ -18,6 +18,7 @@ import { AngularFireAuthModule } from 'angularfire2/auth';
import { AngularFireModule } from 'angularfire2';
import { AngularFirestoreModule } from 'angularfire2/firestore';
import { QuillModule } from 'ngx-quill';
import { EasyPieChartModule } from 'ng2modules-easypiechart';
import { SocialLoginModule, AuthServiceConfig } from 'angularx-social-login';
import {
@@ -81,6 +82,7 @@ import { BoilerplateComponent } from './components/boilerplate/boilerplate.compo
import { BasePageComponent } from './components/base-page/base-page.component';
import { ChatWidgetComponent } from './components/chat-widget/chat-widget.component';
import { ChatService } from 'app/services/chat.service';
import { BytesToHumanPipe } from './pipes/bytes-to-human.pipe';
const cookieConfig: NgcCookieConsentConfig = {
cookie: {
@@ -130,6 +132,7 @@ export function provideConfig() {
NavbarComponent,
FilterEntryPipe,
BytesToHumanPipe,
OrderByPipe,
PrettyPrintPipe,
EntryListItemComponent,
@@ -148,7 +151,8 @@ export function provideConfig() {
SideOverlayComponent,
BoilerplateComponent,
BasePageComponent,
ChatWidgetComponent
ChatWidgetComponent,
BytesToHumanPipe
],
imports: [
BrowserModule,
@@ -172,6 +176,7 @@ export function provideConfig() {
InlineEditorModule,
MomentModule,
QuillModule,
EasyPieChartModule,
ModalModule.forRoot(),
ProgressbarModule.forRoot(),
ToastyModule.forRoot(),

View File

@@ -2,10 +2,17 @@
<h2 class="content-heading text-black">Personal Details</h2>
<div class="row items-push">
<div class="col-lg-3">
<p class="text-muted">
Tell us as much or as little as you like, nobody sees this anyway
</p>
<h2>Avatar Image</h2>
<a class="block block-link-shadow text-right" href="javascript:void(0)">
<div class="block-content block-content-full clearfix">
<div class="chart js-pie-chart pie-chart js-pie-chart-enabled" data-percent="60" #usageChart *ngIf="limit">
<span>{{limit.storageUsed | bytesToHuman}}
<br>
<small class="text-muted">of {{limit.storageQuota| bytesToHuman}}</small>
</span>
</div>
<p class="font-w600 text-center">Storage space used</p>
</div>
</a>
<div class="mb-15 animated fadeIn">
<div class="col-sm-6 col-xl-4">
<div class="options-container text-center">
@@ -44,7 +51,7 @@
<div class="col-6">
<label for="crypto-settings-street-1">Slug</label>
<div class="input-group">
<input type="text" class="form-control" id="slug" name="slug" required="" [(ngModel)]="profile.slug" (input)="onSlugChanged(profile.slug)">
<input type="text" class="form-control" id="slug-box" name="slug-box" autocomplete="off" required="" [(ngModel)]="profile.slug" (input)="onSlugChanged(profile.slug)">
<div class="input-group-append">
<span class="input-group-text">
<i class="fa fa-asterisk"></i>

View File

@@ -2,20 +2,35 @@ import { ProfileService } from 'app/services/profile.service';
import { Store } from '@ngrx/store';
import { ApplicationState } from './../../store/index';
import { Observable } from 'rxjs/Observable';
import { Component, OnInit, ViewChild, ElementRef } from '@angular/core';
import {
Component,
OnInit,
ViewChild,
ElementRef,
AfterViewInit,
NgZone,
ViewChildren,
ViewContainerRef
} from '@angular/core';
import { ProfileModel } from 'app/models/profile.model';
import * as fromProfile from 'app/reducers';
import * as fromProfileActions from 'app/actions/profile.actions';
import { Router } from '@angular/router';
import { ImageService } from 'app/services/image.service';
import { BasePageComponent } from '../base-page/base-page.component';
import { ProfileLimitsModel } from 'app/models/profile.limits';
import { fromEvent } from 'rxjs';
import { map, filter, debounceTime, distinctUntilChanged, switchMap } from 'rxjs/operators';
declare let jQuery: any;
@Component({
selector: 'app-profile',
templateUrl: './profile.component.html',
styleUrls: ['./profile.component.css']
})
export class ProfileComponent extends BasePageComponent implements OnInit {
export class ProfileComponent extends BasePageComponent
implements AfterViewInit {
profile$: Observable<ProfileModel>;
originalSlug: string;
@@ -27,12 +42,19 @@ export class ProfileComponent extends BasePageComponent implements OnInit {
_imageFileBuffer: File;
@ViewChild('fileInput') fileInput: ElementRef;
limits$: Observable<ProfileLimitsModel>;
private usageChart: ElementRef;
@ViewChildren('usageChart', { read: ViewContainerRef })
viewContainerRefs;
limit: any;
constructor(
private _store: Store<ApplicationState>,
private _service: ProfileService,
private _imageService: ImageService,
private _router: Router
private _router: Router,
private _zone: NgZone
) {
super();
this.profile$ = _store.select(fromProfile.getProfile);
@@ -41,7 +63,46 @@ export class ProfileComponent extends BasePageComponent implements OnInit {
this.image.src = p.profileImage;
});
}
ngOnInit() {}
ngAfterViewInit() {
this._service.getLimits().subscribe((l) => {
this.limit = l;
this.viewContainerRefs.changes.subscribe((r) => {
if (this.viewContainerRefs.length !== 0) {
const el = r.first.element.nativeElement;
this._zone.runOutsideAngular(() => {
jQuery(el).easyPieChart({
easing: 'easeOutBounce',
onStep: function(from, to, percent) {
jQuery(el)
.find('.percent')
.text(Math.round(percent));
},
barColor: jQuery(this).attr('data-rel'),
trackColor: 'rgba(0,0,0,0)',
size: 84,
scaleLength: 0,
animation: 2000,
lineWidth: 9,
lineCap: 'round'
});
});
}
});
});
const searchBox = document.getElementById('slug-box');
const typeahead = fromEvent(searchBox, 'input').pipe(
map((e: KeyboardEvent) => e.target.value),
filter((text) => text.length > 2),
debounceTime(10),
distinctUntilChanged(),
switchMap((v) => this.onSlugChanged(v))
);
typeahead.subscribe((data) => {
// Handle the data from the API
});
}
private _parseImageData(file: File) {
const myReader: FileReader = new FileReader();
@@ -52,10 +113,10 @@ export class ProfileComponent extends BasePageComponent implements OnInit {
};
myReader.readAsDataURL(file);
}
onSlugChanged(slug: string) {
onSlugChanged(slug: string) : boolean {
this._service.checkSlug(slug).subscribe((v) => {
console.log('profile.component.ts', 'onSlugChanged', v);
if (v.status == 404) this.slugError = '';
if (v) this.slugError = '';
else this.slugError = 'Slug already exists';
});
}

View File

@@ -0,0 +1,7 @@
import { ProfileModel } from 'app/models/profile.model';
export class ProfileLimitsModel {
user: ProfileModel;
storageQuota: number;
storageUsed: number;
}

View File

@@ -0,0 +1,19 @@
import { Pipe, PipeTransform } from '@angular/core';
@Pipe({
name: 'bytesToHuman'
})
export class BytesToHumanPipe implements PipeTransform {
sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
transform(bytes: number, args?: any): any {
if (bytes == 0) return '0 Bytes';
const k = 1024,
dm = 0,
sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'],
i = Math.floor(Math.log(bytes) / Math.log(k));
return (
parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]
);
}
}

View File

@@ -3,9 +3,9 @@ import { Injectable } from '@angular/core';
import { Observable } from 'rxjs/Observable';
import { ProfileModel } from 'app/models/profile.model';
import 'rxjs/add/operator/map';
import { Profile } from 'selenium-webdriver/firefox';
import { HttpClient } from '@angular/common/http';
import { PodnomsAuthService } from './podnoms-auth.service';
import { ProfileLimitsModel } from 'app/models/profile.limits';
@Injectable()
export class ProfileService {
@@ -38,9 +38,9 @@ export class ProfileService {
);
}
checkSlug(slug): Observable<Response> {
checkSlug(slug): Observable<boolean> {
console.log('profile.service.ts', 'checkSlug', slug);
return this._http.get<Response>(
return this._http.get<boolean>(
environment.API_HOST + '/profile/checkslug/' + slug
);
}
@@ -50,4 +50,9 @@ export class ProfileService {
null
);
}
getLimits(): Observable<ProfileLimitsModel> {
return this._http.get<ProfileLimitsModel>(
environment.API_HOST + '/profile/limits'
);
}
}

View File

@@ -9,7 +9,9 @@
"emitDecoratorMetadata": true,
"experimentalDecorators": true,
"target": "es5",
"typeRoots": ["node_modules/@types"],
// "target": "es2015",
"lib": ["es2017", "dom"]
"lib": ["es2017", "dom"],
"plugins": [{ "name": "tslint-language-service" }]
}
}

View File

@@ -18,13 +18,14 @@ namespace PodNoms.Api.Controllers {
public class ProfileController : BaseAuthController {
public IUnitOfWork _unitOfWork { get; }
public IMapper _mapper { get; }
private readonly IEntryRepository _entryRepository;
public ProfileController(IMapper mapper, IUnitOfWork unitOfWork,
IEntryRepository entryRepository,
UserManager<ApplicationUser> userManager, IHttpContextAccessor contextAccessor)
: base(contextAccessor, userManager) {
this._entryRepository = entryRepository;
this._mapper = mapper;
this._unitOfWork = unitOfWork;
}
@@ -45,13 +46,23 @@ namespace PodNoms.Api.Controllers {
}
[HttpGet("checkslug/{slug}")]
public async Task<IActionResult> CheckSlug(string slug) {
public async Task<ActionResult<bool>> CheckSlug(string slug) {
var slugValid = await _userManager.CheckSlug(slug);
return Ok(slugValid);
}
if (slugValid)
return NotFound();
return Ok();
[HttpGet("limits")]
public async Task<ActionResult<ProfileLimitsViewModel>> GetProfileLimits() {
var entries = await _entryRepository.GetAllForUserAsync(_applicationUser.Id);
var user = _mapper.Map<ApplicationUser, ProfileViewModel>(_applicationUser);
var sum = entries.Select(x => x.AudioFileSize)
.Sum();
var vm = new ProfileLimitsViewModel {
StorageQuota = 5368709120, //5Gb
StorageUsed = sum,
User = user
};
return Ok(vm);
}
}
}

View File

@@ -10,6 +10,6 @@ namespace PodNoms.Api.Models.ViewModels {
public string ImageUrl { get; set; }
public string ThumbnailUrl { get; set; }
public string RssUrl { get; set; }
// public ICollection<PodcastEntryViewModel> PodcastEntries { get; set; }
public ICollection<PodcastEntryViewModel> PodcastEntries { get; set; }
}
}

View File

@@ -0,0 +1,10 @@
using System;
using System.Numerics;
namespace PodNoms.Api.Models.ViewModels {
public class ProfileLimitsViewModel {
public decimal StorageQuota { get; set; }
public decimal StorageUsed { get; set; }
public ProfileViewModel User { get; set; }
}
}

View File

@@ -26,7 +26,7 @@ namespace PodNoms.Api.Services.Jobs {
this._logger = logger.CreateLogger<ClearOrphanAudioJob>();
}
public async Task Execute() {
public async Task<bool> Execute() {
try {
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(_storageSettings.ConnectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
@@ -49,9 +49,11 @@ namespace PodNoms.Api.Services.Jobs {
}
}
await this._mailSender.SendEmailAsync("fergal.moran@gmail.com", $"ClearOrphanAudioJob: Complete {blobCount}", string.Empty);
return true;
} catch (Exception ex) {
_logger.LogError($"Error clearing orphans\n{ex.Message}");
}
return false;
}
}
}

View File

@@ -2,6 +2,6 @@ using System.Threading.Tasks;
namespace PodNoms.Api.Services.Jobs {
public interface IJob {
Task Execute();
Task<bool> Execute();
}
}

View File

@@ -12,6 +12,10 @@ namespace PodNoms.Api.Services.Jobs {
RecurringJob.AddOrUpdate<UpdateYouTubeDlJob>(x => x.Execute(), Cron.Daily(1, 30));
BackgroundJob.Schedule<ProcessPlaylistsJob>(x => x.Execute(3), TimeSpan.FromSeconds(1));
RecurringJob.AddOrUpdate<ProcessPlaylistsJob>(x => x.Execute(), Cron.Daily(2));
BackgroundJob.Schedule<ProcessRemoteAudioFileAttributesJob>(
x => x.Execute(),
TimeSpan.FromSeconds(Int16.MaxValue));
}
}
}

View File

@@ -34,14 +34,15 @@ namespace PodNoms.Api.Services.Jobs {
this._logger = logger;
}
[Mutex("ProcessPlaylistItemJob")]
public async Task Execute() {
public async Task<bool> Execute() {
var items = await _playlistRepository.GetUnprocessedItems();
foreach (var item in items) {
await ExecuteForItem(item.VideoId, item.Playlist.Id);
}
return true;
}
[Mutex("ProcessPlaylistItemJob")]
public async Task ExecuteForItem(string itemId, int playlistId) {
public async Task<bool> ExecuteForItem(string itemId, int playlistId) {
var item = await _playlistRepository.GetParsedItem(itemId, playlistId);
if (item != null && !string.IsNullOrEmpty(item.VideoType) &&
(item.VideoType.Equals("youtube") || item.VideoType.Equals("mixcloud"))) {
@@ -80,9 +81,11 @@ namespace PodNoms.Api.Services.Jobs {
}
} else {
_logger.LogError($"Processing playlist item {itemId} failed");
return false;
}
}
}
return true;
}
}
}

View File

@@ -37,17 +37,20 @@ namespace PodNoms.Api.Services.Jobs {
this._logger = logger.CreateLogger<ProcessPlaylistsJob>();
}
public async Task Execute() {
public async Task<bool> Execute() {
var playlists = _playlistRepository.GetAll()
.ToList();
foreach (var playlist in playlists) {
await Execute(playlist.Id);
}
return true;
}
public async Task Execute(int playlistId) {
public async Task<bool> Execute(int playlistId) {
try {
var playlist = await _playlistRepository.GetAsync(playlistId);
if (playlist == null)
return false;
var resultList = new List<ParsedItemResult>();
var downloader = new AudioDownloader(playlist.SourceUrl, _helpersSettings.Downloader);
@@ -76,9 +79,11 @@ namespace PodNoms.Api.Services.Jobs {
}
}
}
return true;
} catch (Exception ex) {
_logger.LogError(ex.Message);
}
return false;
}
}
}

View File

@@ -0,0 +1,43 @@
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using PodNoms.Api.Models.Settings;
using PodNoms.Api.Persistence;
using PodNoms.Api.Services.Storage;
namespace PodNoms.Api.Services.Jobs {
public class ProcessRemoteAudioFileAttributesJob : IJob {
private readonly IEntryRepository _entryRepository;
private readonly IFileUtilities _fileUtilities;
private readonly ILogger<ProcessRemoteAudioFileAttributesJob> _logger;
private readonly IUnitOfWork _unitOfWork;
public ProcessRemoteAudioFileAttributesJob(IEntryRepository entryRepository,
IFileUtilities fileUtilities, ILogger<ProcessRemoteAudioFileAttributesJob> logger,
IUnitOfWork unitOfWork) {
this._logger = logger;
this._entryRepository = entryRepository;
this._fileUtilities = fileUtilities;
this._unitOfWork = unitOfWork;
}
public async Task<bool> Execute() {
var entries = await _entryRepository.GetAll()
.ToListAsync();
foreach (var entry in entries) {
string[] parts = entry.AudioUrl.Split("/");
if (parts.Length == 2) {
_logger.LogInformation($"Processing remote: {entry.AudioUrl}");
var size = await _fileUtilities.GetRemoteFileSize(
parts[0], parts[1]);
if (size != -1) {
entry.AudioFileSize = size;
}
}
}
await _unitOfWork.CompleteAsync();
return false;
}
}
}

View File

@@ -10,12 +10,12 @@ namespace PodNoms.Api.Services.Jobs {
private readonly IMailSender _sender;
private readonly ILogger _logger;
public UpdateYouTubeDlJob(IMailSender sender, ILogger<UpdateYouTubeDlJob> logger){
public UpdateYouTubeDlJob(IMailSender sender, ILogger<UpdateYouTubeDlJob> logger) {
this._sender = sender;
this._logger = logger;
}
public async Task Execute() {
public async Task<bool> Execute() {
_logger.LogInformation("Updating YoutubeDL");
var yt = new YoutubeDL();
@@ -24,6 +24,7 @@ namespace PodNoms.Api.Services.Jobs {
var results = await _sender.SendEmailAsync("fergal.moran@gmail.com", "PodNoms: UpdateYouTubeDlJob completed", "As you were");
_logger.LogInformation($"{results}");
return true;
}
}
}

View File

@@ -42,7 +42,8 @@ namespace PodNoms.Api.Services.Processor {
localFile = entry.AudioUrl;
if (File.Exists(localFile)) {
var fileName = new FileInfo(localFile).Name;
FileInfo fileInfo = new FileInfo(localFile);
var fileName = fileInfo.Name;
await _fileUploader.UploadFile(localFile, _audioStorageSettings.ContainerName, fileName,
"application/mpeg",
async (p, t) => {
@@ -60,6 +61,7 @@ namespace PodNoms.Api.Services.Processor {
entry.Processed = true;
entry.ProcessingStatus = ProcessingStatus.Processed;
entry.AudioUrl = $"{_audioStorageSettings.ContainerName}/{fileName}";
entry.AudioFileSize = fileInfo.Length;
await _unitOfWork.CompleteAsync();
await _sendProcessCompleteMessage(entry);
return true;

View File

@@ -15,6 +15,7 @@ using PodNoms.Api.Services.Realtime;
using PodNoms.Api.Utils.Extensions;
namespace PodNoms.Api.Services.Storage {
internal class AzureFileUploader : IFileUploader {
private readonly StorageSettings _settings;
public AzureFileUploader(IOptions<StorageSettings> settings, ILoggerFactory logger) {

View File

@@ -0,0 +1,30 @@
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using PodNoms.Api.Models.Settings;
namespace PodNoms.Api.Services.Storage {
internal class AzureFileUtilities : IFileUtilities {
private readonly StorageSettings _settings;
public AzureFileUtilities(IOptions<StorageSettings> settings, ILoggerFactory logger) {
this._settings = settings.Value;
}
public async Task<long> GetRemoteFileSize(string containerName, string fileName) {
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(_settings.ConnectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
var exists = await container.ExistsAsync();
if (exists) {
CloudBlockBlob blob = container.GetBlockBlobReference(fileName);
exists = await blob.ExistsAsync();
if (exists) {
await blob.FetchAttributesAsync();
return blob.Properties.Length;
}
}
return -1;
}
}
}

View File

@@ -0,0 +1,7 @@
using System.Threading.Tasks;
namespace PodNoms.Api.Services.Storage {
public interface IFileUtilities {
Task<long> GetRemoteFileSize(string containerName, string fileName);
}
}

View File

@@ -255,6 +255,7 @@ namespace PodNoms.Api {
services.AddScoped<IAudioUploadProcessService, AudioUploadProcessService>();
services.AddScoped<ISupportChatService, SupportChatService>();
services.AddScoped<IMailSender, MailgunSender>();
services.AddScoped<IFileUtilities, AzureFileUtilities>();
services.AddScoped<YouTubeParser>();
services.AddScoped<MixcloudParser>();
services.AddScoped<SlackSupportClient>();