Added rabbit mq for waveform creation.

core/tasks.py

@@ -0,0 +1,22 @@
+import shutil
+from celery.task import task
+import os
+from core.utils.waveform import generate_waveform
+from dss import settings
+
+"""
+    name='core.tasks.create_waveform_task'
+"""
+@task()
+def create_waveform_task(in_file, uid):
+    out_file = os.path.join(settings.MEDIA_ROOT, 'waveforms/%s.png' % uid)
+    print "Creating waveform \n\tIn: %s\n\tOut: %s" % (in_file, out_file)
+    generate_waveform(in_file, out_file)
+    if os.path.isfile(out_file):
+        print "Waveform generated successfully"
+        file, extension = os.path.splitext(in_file)
+        new_file = in_file.replace('cache', 'mixes')
+        print "Moving cache audio clip from %s to %s" % (in_file, new_file)
+        shutil.move(in_file, new_file)
+    else:
+        print "Outfile is missing"

core/tasks/__init__.py

@@ -1 +0,0 @@
-__author__ = 'fergalm'

core/tasks/waveform.py

@@ -1,8 +0,0 @@
-from celery.task import task
-from core.utils.waveform import generate_waveform
-
-@task(name='dss.create_waveform_task')
-def create_waveform_task(in_file, out_file):
-    generate_waveform(in_file, out_file)
-
-

core/utils/waveform.py

@@ -4,8 +4,6 @@ import uuid
 import os
 from dss import settings
 
-__author__ = 'fergalm'
-
 def generate_waveform(input_file, output_file):
     print "Generating waveform"
     try:

dss/settings.py

@@ -37,6 +37,7 @@ USE_TZ = False
 
 SITE_ROOT = here('')
 MEDIA_ROOT = here('media')
+CACHE_ROOT = here('media/cache')
 MEDIA_URL = '/media/'
 
 STATIC_ROOT = localsettings.STATIC_ROOT if hasattr(localsettings, 'STATIC_ROOT') else ''
@@ -135,6 +136,7 @@ INSTALLED_APPS = (
     'avatar',
     'notification',
     'spa',
+    'core',
     'allauth',
     'allauth.account',
     'allauth.socialaccount',

requirements.txt

@@ -1,7 +1,4 @@
-PIL
 Django>=1.4
-MySQL-python>=1.2.3
-pillow>=1.7.7
 beautifulsoup4>=4.1.1
 decorator>=3.3.3
 django-allauth>=0.7.0
@@ -18,4 +15,5 @@ git+git://github.com/maraujop/django-crispy-forms.git#django-crispy-forms
 git+git://github.com/tschellenbach/Django-facebook.git#django-facebook
 django-grappelli
 humanize
-sorl-thumbnail>=11.12
+django-debug-toolbar
+sorl-thumbnail

spa/ajax.py

@@ -9,12 +9,12 @@ from django.utils import simplejson
 from django.views.decorators.csrf import csrf_exempt
 import os
 from core.utils import live
-from dss import localsettings
+from dss import localsettings, settings
 from spa.models import UserProfile, MixFavourite
 from spa.models.Mix import Mix
 from spa.models.Comment import Comment
 from spa.models.MixLike import MixLike
-from tasks.waveform import create_waveform_task
+from core.tasks import create_waveform_task
 import logging
 
 logger = logging.getLogger(__name__)
@@ -187,20 +187,18 @@ def upload_mix_file_handler(request):
         if 'Filedata' in request.FILES and 'upload-hash' in request.POST:
             f = request.FILES['Filedata']
             fileName, extension = os.path.splitext(f.name)
-            in_file = 'media/cache/%s' % (request.POST['upload-hash'])
+            uid = request.POST['upload-hash']
+            in_file = '%s/%s%s' % (settings.CACHE_ROOT, uid, extension)
             with open(in_file, 'wb+') as destination:
                 for chunk in f.chunks():
                     destination.write(chunk)
-
             try:
-                create_waveform_task.delay(in_file=in_file)
+                create_waveform_task.delay(in_file=in_file, uid=uid)
             except Exception, ex:
                 logger.exception("Error starting waveform generation task: %s" % ex.message)
 
-
         return HttpResponse(_get_json("Success"))
     except Exception, ex:
         logger.exception("Error uploading mix")
 
     return HttpResponse(_get_json("Failed"))
-

spa/api/v1/MixResource.py

@@ -18,10 +18,11 @@ class MixResource(BackboneCompatibleResource):
         authorization = Authorization()
 
     def obj_create(self, bundle, request=None, **kwargs):
-        file_name = "media/mixes/%s" % bundle.data['upload-hash']
+        file_name = "mixes/%s.%s" % (bundle.data['upload-hash'], bundle.data['upload-extension'])
+        uid = bundle.data['upload-hash']
         bundle.data['user'] = request.user.get_profile()
         return super(MixResource, self).obj_create(bundle, request, user=request.user.get_profile(),
-            local_file=file_name)
+            local_file=file_name, uid=uid)
 
     def obj_get_list(self, request=None, **kwargs):
         sort = 'latest'

spa/management/commands/waveforms.py

@@ -1,9 +1,8 @@
 import os
 from dss import settings
 from core.utils.waveform import generate_waveform
-from spa.models.Mix import Mix
-from spa.models.Release import ReleaseAudio
 from django.core.management.base import NoArgsCommand
+from spa.models.Mix import Mix
 
 class Command(NoArgsCommand):
     help = "Generate all outstanding waveforms"
@@ -14,15 +13,15 @@ class Command(NoArgsCommand):
                 print "Found missing waveform"
                 generate_waveform(local_file, file)
 
+    def handle(self, *args, **options):
+        for guid in args:
+            output_file = 'waveforms/%s.png' % guid
+            local_file =  '%s/%s' % (self.CACHE_ROOT, guid)
+            if os.path.isfile(local_file):
+                self._check_file(local_file, output_file)
+
     def handle_noargs(self, **options):
         print "Generating waveforms for mix"
-        objects = Mix.objects.all()
+        unprocessed = Mix.objects.filter(waveform_generated=False)
-        for object in objects:
+        for mix in unprocessed:
-            output_file = 'waveforms/mix/%d.png' % object.pk
+            pass
-            self._check_file(object.local_file.file.name, output_file)
-            
-        print "Generating waveforms for release"
-        objects = ReleaseAudio.objects.all()
-        for object in objects:
-            output_file = 'waveforms/release/%d.png' % object.pk
-            self._check_file(object.local_file.file.name, output_file)

spa/models/Mix.py

@@ -29,6 +29,7 @@ class Mix(_BaseModel):
     is_active = models.BooleanField(default=True)
     user = models.ForeignKey(UserProfile, editable=False)
     waveform_generated = models.BooleanField(default=False)
+    uid = models.CharField(max_length=38, blank=True)
 
     def __unicode__(self):
         return self.title
@@ -40,10 +41,10 @@ class Mix(_BaseModel):
         return '/mix/%i' % self.id
 
     def get_waveform_path(self):
-        return os.path.join(settings.MEDIA_ROOT, "waveforms/mix/", "%d.%s" % (self.id, "png"))
+        return os.path.join(settings.MEDIA_ROOT, "waveforms/", "%s.%s" % (self.uid, "png"))
 
     def get_waveform_url(self):
-        return settings.MEDIA_URL + 'waveforms/mix/%d.%s' % (self.id, "png")
+        return settings.MEDIA_URL + 'waveforms/%s.%s' % (self.uid, "png")
 
     def get_image(self):
         try:

spa/models/MixPlay.py

@@ -2,7 +2,4 @@ from django.db import models
 from spa.models._Activity import _Activity
 
 class MixPlay(_Activity):
-    class Meta:
-        app_label = 'spa'
-
     mix = models.ForeignKey('spa.Mix', related_name='plays')

spa/models/_Activity.py

@@ -4,4 +4,4 @@ from spa.models._BaseModel import _BaseModel
 
 class _Activity(_BaseModel):
     date = models.DateTimeField(auto_now=True)
-    user = models.ForeignKey(User, null=True    )
+    user = models.ForeignKey(User, null=True)

spa/urls.py

@@ -24,9 +24,6 @@ urlpatterns = django.conf.urls.patterns(
     '',
     url(r'^$', 'spa.views.app', name='home'),
     url(r'^upload', 'spa.views.app', name='home'),
-    url(r'^mix/upload/progress/$',
-        'spa.views.upload_progress',
-        name='upload_progress'),
     url(r'^tpl/(?P<template_name>\w+)/$', 'spa.templates.get_template'),
     url(r'^js/(?P<template_name>\w+)/$', 'spa.templates.get_javascript'),
     url(r'^tplex/(?P<template_name>\w+)/$', 'spa.templates.get_template_ex'),

spa/views.py

@@ -11,20 +11,4 @@ def app(request):
 def upload(request):
     return render_to_response("inc/upload.html", context_instance=RequestContext(request))
 
-@login_required
-def upload_progress(request):
-    """
-    Return JSON object with information about the progress of an upload.
-    """
-    if 'HTTP_X_PROGRESS_ID' in request.META:
-        progress_id = request.META['HTTP_X_PROGRESS_ID']
-        from django.utils import simplejson
 
-        cache_key = "%s_%s" % (request.META['REMOTE_ADDR'], progress_id)
-        data = cache.get(cache_key)
-        json = simplejson.dumps(data)
-        print >> sys.stderr, json
-        return HttpResponse(json)
-    else:
-        logging.error("Received progress report request without X-Progress-ID header. request.META: %s" % request.META)
-        return HttpResponseBadRequest('Server Error: You must provide X-Progress-ID header or query param.')

static/js/app/views/mix.js

@@ -142,17 +142,19 @@ window.MixCreateView = Backbone.View.extend({
                 'upload-hash' : this.guid
             },
             'onAddQueueItem':function (file) {
+                $('#upload-extension', this.el).val(file.name.split('.').pop());
                 $('#mix-details', this.el).show();
             },
             'onProgress':function (file, e) {
             }
         });
-        //$('#mix-details', this.el).hide();
+        $('#mix-details', this.el).hide();
         $('.upload-hash', this.el).val(this.guid);
         return this;
     },
     saveChanges:function () {
         this.model.set('upload-hash', this.guid);
+        this.model.set('upload-extension', $('#upload-extension', this.el).val());
         this.model.save(
             null, {
                 success:function () {

tasks/__init__.py

@@ -1 +0,0 @@
-__author__ = 'fergalm'

tasks/waveform.py

@@ -1,50 +0,0 @@
-import os
-import subprocess
-import traceback
-import uuid
-from dss import settings
-from celery.task.base import task
-import spa
-
-@task(name='www.create_waveform_task')
-def create_waveform_task(in_file, id=None):
-    try:
-        working_file = "/tmp/%s.wav" % (uuid.uuid1())
-        out_file = os.path.join(settings.MEDIA_ROOT, 'waveforms/%i.png' % id)
-        try:
-            print "Starting decode"
-            p = subprocess.call(["/usr/bin/lame", "--decode", in_file, working_file])
-            print "Finished decode"
-            if os.path.exists(working_file):
-                print "Starting waveform generation"
-                subprocess.call(["waveformgen",  "-m", "-l", "-i", working_file, "-o", out_file])
-
-                if os.path.isfile(out_file):
-                    os.remove(working_file)
-                    print "Generated waveform"
-                    return out_file
-                else:
-                    print "Failed generating waveform: %s" % out_file
-            else:
-                print "Unable to find working file, did LAME succeed?"
-                return ""
-        except:
-            print "Error generating waveform"
-            traceback.print_exc()
-
-    except:
-        print "Error generating waveform"
-        traceback.print_exc()
-
-
-@task(name='www.gmw')
-def gmw():
-    objects = spa.models.mix.Mix.objects.all()
-
-    for object in objects:
-        file = os.path.join(settings.MEDIA_ROOT, 'waveforms/%i.png' % object.pk)
-        if not os.path.isfile(file):
-            print "Found missing waveform"
-            create_waveform_task(object.pk, object.local_file.file.name)
-
-

templates/views/MixCreateView.html

@@ -12,6 +12,7 @@
     <form class="form-horizontal">
         {% csrf_token %}
         <input type="hidden" class="upload-hash" id="upload-hash-details">
+        <input type="hidden" id="upload-extension">
         <legend>Mix details</legend>
         <div class="control-group">
             <label class="control-label">Title</label>