diff --git a/api/views.py b/api/views.py index 53349de..63cdc3f 100755 --- a/api/views.py +++ b/api/views.py @@ -18,7 +18,7 @@ from rest_framework.status import HTTP_202_ACCEPTED, HTTP_401_UNAUTHORIZED, HTTP from api import serializers from dss import settings -from spa.tasks import create_waveform_task, archive_mix_task +from spa.tasks import create_waveform_task, upload_to_cdn_task from spa.models.genre import Genre from spa.models.activity import ActivityPlay from spa.models.mix import Mix @@ -179,12 +179,18 @@ class PartialMixUploadView(views.APIView): # Chain the waveform & archive tasks together # Probably not the best place for them but will do for now - # First argument to archive_mix_task is not specified as it is piped from create_waveform_task + # First argument to upload_to_cdn_task is not specified as it is piped from create_waveform_task logger.debug("Processing input_file: {0}".format(input_file)) logger.debug("Connecting to broker: {0}".format(settings.BROKER_URL)) + + from celery import group (create_waveform_task.s(input_file, uid) | - archive_mix_task.s(filetype='mp3', uid=uid)).delay() + group( + upload_to_cdn_task.s(filetype='mp3', uid=uid, container_name='mixes'), + upload_to_cdn_task.s(filetype='png', uid=uid, container_name='waveforms') + ) + ).delay() logger.debug("Waveform task started") except Exception, ex: diff --git a/core/utils/cdn.py b/core/utils/cdn.py index 09f5a80..4ea6d82 100755 --- a/core/utils/cdn.py +++ b/core/utils/cdn.py @@ -8,13 +8,13 @@ from libcloud.storage.types import Provider from libcloud.storage.providers import get_driver -def upload_to_azure(in_file, filetype, uid): +def upload_to_azure(in_file, filetype, uid, container_name=settings.AZURE_CONTAINER): if os.path.isfile(in_file): print "Uploading file for: %s" % in_file file_name = "%s.%s" % (uid, filetype) cls = get_driver(Provider.AZURE_BLOBS) driver = cls(settings.AZURE_ACCOUNT_NAME, settings.AZURE_ACCOUNT_KEY) - container = driver.get_container(container_name=settings.AZURE_CONTAINER) + container = driver.get_container(container_name) with open(in_file, 'rb') as iterator: obj = driver.upload_object_via_stream( @@ -47,3 +47,13 @@ def set_azure_details(blob_name, download_name): print "No blob found for: %s" % download_name except Exception, ex: print "Error processing blob %s: %s" % (download_name, ex.message) + +def file_exists(url): + import httplib + from urlparse import urlparse + p = urlparse(url) + c = httplib.HTTPConnection(p.netloc) + c.request("HEAD", p.path) + + r = c.getresponse() + return r.status == 200 diff --git a/dss/settings.py b/dss/settings.py index 40e662e..6dc6b9d 100755 --- a/dss/settings.py +++ b/dss/settings.py @@ -107,6 +107,7 @@ INSTALLED_APPS = ( 'django_gravatar', 'corsheaders', 'sorl.thumbnail', + 'djcelery', 'spa', 'gunicorn', 'spa.signals', diff --git a/spa/models/mix.py b/spa/models/mix.py index 1ed76d6..5a70c03 100755 --- a/spa/models/mix.py +++ b/spa/models/mix.py @@ -1,8 +1,7 @@ -import os import rfc822 -import urlparse -from django.utils.encoding import smart_str +import os +from django.utils.encoding import smart_str from sorl.thumbnail import get_thumbnail from django.contrib.sites.models import Site from django.db import models @@ -19,8 +18,7 @@ from dss import settings, localsettings from spa.models.userprofile import UserProfile from spa.models.basemodel import BaseModel from core.utils.file import generate_save_file_name -from PIL import Image -import glob +from core.utils import cdn class Engine(Engine): @@ -104,7 +102,7 @@ class Mix(BaseModel): self.clean_image('mix_image', Mix) # Check for the unlikely event that the waveform has been generated - if os.path.isfile(self.get_waveform_path()): + if cdn.file_exists('{0}{1}.png'.format(localsettings.WAVEFORM_URL, self.uid)): self.waveform_generated = True try: self.duration = mp3_length(self.get_absolute_path()) diff --git a/spa/tasks.py b/spa/tasks.py index a823dc1..c2bfeaa 100755 --- a/spa/tasks.py +++ b/spa/tasks.py @@ -15,31 +15,27 @@ from dss import settings @task(time_limit=3600) def create_waveform_task(in_file, uid): - out_file = os.path.join(settings.MEDIA_ROOT, 'waveforms/%s.png' % uid) + out_file = os.path.join(settings.CACHE_ROOT, 'waveforms/%s.png' % uid) print "Creating waveform \n\tIn: %s\n\tOut: %s" % (in_file, out_file) generate_waveform(in_file, out_file) if os.path.isfile(out_file): print "Waveform generated successfully" - out_file, extension = os.path.splitext(in_file) - new_file = os.path.join(settings.MEDIA_ROOT, "mixes", "%s%s" % (uid, extension)) - print "Moving cache audio clip from %s to %s" % (in_file, new_file) - shutil.move(in_file, new_file) - print "Uid: %s" % uid waveform_generated_signal.send(sender=None, uid=uid) - return new_file + return out_file else: print "Outfile is missing" -@task(time_limit=3600) -def archive_mix_task(in_file, filetype, uid): +@task(timse_limit=3600) +def upload_to_cdn_task(in_file, filetype, uid, container_name): + source_file = os.path.join(settings.CACHE_ROOT, '{0}/{1}.{2}'.format(container_name, uid, filetype)) print "Sending {0} to azure".format(uid) try: - upload_to_azure(in_file, filetype, uid) + upload_to_azure(source_file, filetype, uid, container_name) + return source_file except Exception, ex: print "Unable to upload: {0}".format(ex.message) - @task def update_geo_info_task(ip_address, profile_id): try: