Merge tag 'admin_delete_comments' into develop

Finished
This commit is contained in:
Fergal Moran
2016-06-29 18:48:21 +01:00
19 changed files with 173 additions and 145 deletions

3
.gitignore vendored
View File

@@ -16,6 +16,7 @@ dss/celery_settings.py
dss/devsettings.py
dss.conf
dss/debugsettings.py
logs/
mysql
test.py
mysql2pgsql.yml
@@ -34,4 +35,4 @@ reload
reset
__krud/
celerybeat-schedule
private/
private/

View File

@@ -1,26 +1,12 @@
FROM python:latest
ENV PYTHONBUFFERED 1
FROM fergalmoran/django
RUN mkdir /code
RUN mkdir /srv/logs
RUN mkdir /files
RUN mkdir /files/static
RUN mkdir /files/media
RUN mkdir /files/cache
RUN mkdir /files/cache/mixes
RUN mkdir /files/cache/waveforms
RUN mkdir /files/tmp
WORKDIR /code
ADD requirements.txt /code/
ADD . /code/
RUN apt-get update && apt-get install -y sox lame vim ccze node npm \
libboost-program-options-dev libsox-fmt-mp3 postgresql-client rsync openssh-client
RUN npm install -g yuglify
RUN pip install -r requirements.txt
RUN mkdir /files/static
RUN mkdir /files/media
RUN mkdir /files/cache/mixes
RUN mkdir /files/cache/waveforms
RUN touch /files/tmp/dss.log
RUN adduser --disabled-password --gecos '' djworker
RUN chown djworker /files -R
WORKDIR /code

View File

@@ -1,20 +1,19 @@
import datetime
import json
import logging
from calendar import timegm
from urllib.parse import parse_qsl
import requests
from allauth.socialaccount import models as aamodels
from requests_oauthlib import OAuth1
from rest_framework import parsers
from rest_framework import permissions
from rest_framework import renderers
from rest_framework import parsers, renderers
from rest_framework import status
from rest_framework.authtoken.models import Token
from rest_framework.authtoken.serializers import AuthTokenSerializer
from rest_framework.permissions import AllowAny
from rest_framework.response import Response
from rest_framework.views import APIView
from rest_framework.views import status
from rest_framework_jwt.authentication import JSONWebTokenAuthentication
from rest_framework_jwt.settings import api_settings
from rest_framework_jwt.utils import jwt_payload_handler, jwt_encode_handler
@@ -22,14 +21,6 @@ from dss import settings
from spa.models import UserProfile
from spa.models.socialaccountlink import SocialAccountLink
logger = logging.getLogger('dss')
BACKENDS = {
'google': 'google-oauth2',
'facebook': 'facebook',
'twitter': 'twitter'
}
def _temp_reverse_user(uid, provider, access_token, access_token_secret, payload):
"""
@@ -45,7 +36,7 @@ def _temp_reverse_user(uid, provider, access_token, access_token_secret, payload
sa.access_token_secret = access_token_secret
sa.provider_data = payload
sa.save()
user = UserProfile.objects.get(user__id=sa.user.id)
user = UserProfile.objects.get(id=sa.user.id)
except SocialAccountLink.DoesNotExist:
# try allauth
try:
@@ -70,7 +61,8 @@ def _temp_reverse_user(uid, provider, access_token, access_token_secret, payload
class SocialLoginHandler(APIView):
permission_classes = (permissions.AllowAny,)
"""View to authenticate users through social media."""
permission_classes = (AllowAny,)
def post(self, request):
uid = None
@@ -190,7 +182,8 @@ class ObtainUser(APIView):
'name': request.user.username,
'session': request.user.userprofile.get_session_id(),
'slug': request.user.userprofile.slug,
'userRole': 'user'
'session': request.user.userprofile.get_session_id(),
'userRole': 'user',
})
else:
return Response(status=status.HTTP_401_UNAUTHORIZED)

View File

@@ -34,6 +34,9 @@ class InlineUserProfileSerializer(serializers.ModelSerializer):
profile_image_small = serializers.SerializerMethodField()
profile_image_medium = serializers.SerializerMethodField()
profile_image_header = serializers.SerializerMethodField()
first_name = serializers.ReadOnlyField(source='get_first_name')
last_name = serializers.ReadOnlyField(source='get_last_name')
display_name = serializers.SerializerMethodField()
class Meta:
model = UserProfile
@@ -48,15 +51,12 @@ class InlineUserProfileSerializer(serializers.ModelSerializer):
'profile_image_header',
)
first_name = serializers.ReadOnlyField(source='get_first_name')
last_name = serializers.ReadOnlyField(source='get_last_name')
display_name = serializers.ReadOnlyField(source='get_nice_name')
def get_avatar_image(self, obj):
return obj.get_sized_avatar_image(64, 64)
return obj.get_sized_avatar_image(32, 32)
def get_avatar_image_tiny(self, obj):
return obj.get_sized_avatar_image(64, 64)
return obj.get_sized_avatar_image(32, 32)
def to_representation(self, instance):
if instance.user.is_anonymous():
@@ -72,14 +72,20 @@ class InlineUserProfileSerializer(serializers.ModelSerializer):
return obj.is_follower(self.context['request'].user)
def get_profile_image_small(self, obj):
return obj.get_sized_avatar_image(64, 64)
return obj.get_sized_avatar_image(32, 32)
def get_profile_image_medium(self, obj):
return obj.get_sized_avatar_image(170, 170)
return obj.get_sized_avatar_image(253, 157)
def get_profile_image_header(self, obj):
return obj.get_sized_avatar_image(1200, 150)
def get_display_name(self, obj):
n = obj.get_nice_name()
if not n or n == "":
n = "%s %s".format(obj.first_name, obj.last_name)
return n
class LikeSerializer(serializers.ModelSerializer):
class Meta:
@@ -196,6 +202,18 @@ class MixSerializer(serializers.ModelSerializer):
except UserProfile.DoesNotExist:
pass
genres = self.initial_data['genres']
instance.genres.clear()
for genre in genres:
try:
g = Genre.objects.get(slug=genre.get('slug'))
instance.genres.add(g)
except Genre.DoesNotExist:
""" Possibly allow adding genres here """
pass
validated_data.pop('genres', None)
# get any likes that aren't in passed bundle
if 'downloads' in validated_data:
plays = validated_data['downloads'] or []
@@ -203,12 +221,6 @@ class MixSerializer(serializers.ModelSerializer):
instance.add_play(play)
validated_data.pop('downloads', None)
if 'genres' in validated_data:
genres = validated_data['genres'] or []
for genre in genres:
instance.add_genre(genre)
validated_data.pop('genres', None)
return super(MixSerializer, self).update(instance, validated_data)
except MixUpdateException as ex:
raise ex
@@ -217,7 +229,7 @@ class MixSerializer(serializers.ModelSerializer):
return super(MixSerializer, self).is_valid(raise_exception)
def get_avatar_image(self, obj):
return obj.user.get_sized_avatar_image(64, 64)
return obj.user.get_sized_avatar_image(32, 32)
def get_can_edit(self, obj):
user = self.context['request'].user
@@ -254,6 +266,7 @@ class UserProfileSerializer(serializers.ModelSerializer):
profile_image_small = serializers.SerializerMethodField()
profile_image_medium = serializers.SerializerMethodField()
profile_image_header = serializers.SerializerMethodField()
display_name = serializers.SerializerMethodField(source='get_display_name')
top_tags = serializers.SerializerMethodField()
@@ -304,6 +317,13 @@ class UserProfileSerializer(serializers.ModelSerializer):
pass
return super(UserProfileSerializer, self).update(instance, validated_data)
def get_display_name(self, obj):
n = obj.get_nice_name()
if not n or n == "":
n = "%s %s".format(obj.first_name, obj.last_name)
return n
def get_title(self, obj):
try:
if obj.description:
@@ -346,10 +366,10 @@ class UserProfileSerializer(serializers.ModelSerializer):
values('total', 'description', 'slug')[0:3])
def get_profile_image_small(self, obj):
return obj.get_sized_avatar_image(64, 64)
return obj.get_sized_avatar_image(32, 32)
def get_profile_image_medium(self, obj):
return obj.get_sized_avatar_image(170, 170)
return obj.get_sized_avatar_image(253, 157)
def get_profile_image_header(self, obj):
return obj.get_sized_avatar_image(1200, 150)
@@ -402,8 +422,11 @@ class CommentSerializer(serializers.HyperlinkedModelSerializer):
def get_can_edit(self, obj):
user = self.context['request'].user
if user is not None and obj.user is not None and user.is_authenticated():
return user.is_staff or obj.user.id == user.userprofile.id
if user is not None:
if user.is_staff:
return True
if obj.user is not None and user.is_authenticated():
return obj.user.id == user.userprofile.id
return False
@@ -425,7 +448,7 @@ class HitlistSerializer(serializers.ModelSerializer):
return obj.get_nice_name()
def get_avatar_image(self, obj):
return obj.get_sized_avatar_image(170, 170)
return obj.get_sized_avatar_image(253, 157)
class ActivitySerializer(serializers.HyperlinkedModelSerializer):
@@ -454,7 +477,7 @@ class NotificationSerializer(serializers.HyperlinkedModelSerializer):
from_user = InlineUserProfileSerializer(source='get_from_user', read_only=True)
notification_url = serializers.ReadOnlyField()
verb = serializers.ReadOnlyField()
target = serializers.ReadOnlyField()
target = serializers.SerializerMethodField()
date = serializers.ReadOnlyField()
class Meta:
@@ -475,7 +498,10 @@ class NotificationSerializer(serializers.HyperlinkedModelSerializer):
return settings.DEFAULT_USER_NAME if obj.from_user is None else obj.from_user.get_nice_name()
def get_avatar_image(self, obj):
return settings.DEFAULT_USER_IMAGE if obj.from_user is None else obj.from_user.get_sized_avatar_image(170, 170)
return settings.DEFAULT_USER_IMAGE if obj.from_user is None else obj.get_sized_avatar_image(253, 157)
def get_target(self, obj):
return "/{}/{}".format(obj.to_user.slug, obj.target)
class MessageSerializer(serializers.ModelSerializer):

View File

@@ -1,20 +1,17 @@
from django.conf.urls import patterns, url, include
from rest_framework import permissions
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from rest_framework.routers import DefaultRouter
from rest_framework.views import APIView
from rest_framework.views import status
from rest_framework_jwt.authentication import JSONWebTokenAuthentication
from api import views, auth, helpers
from api.auth import SocialLoginHandler
from rest_framework.views import status
from rest_framework.response import Response
from core.realtime import activity
router = DefaultRouter() # trailing_slash=True)
router.register(r'user', views.UserProfileViewSet)
router.register(r'mix', views.MixViewSet)
router.register(r'notification', views.NotificationViewSet)
router.register(r'hitlist', views.HitlistViewSet)
@@ -26,6 +23,8 @@ router.register(r'shows', views.ShowViewSet, base_name='shows')
router.register(r'blog', views.BlogViewSet, base_name='shows')
router.register(r'playlist', views.PlaylistViewSet, base_name='playlists')
router.register(r'user', views.UserProfileViewSet)
router.register(r'mix', views.MixViewSet)
class DebugView(APIView):
#permission_classes = (IsAuthenticated,)

View File

@@ -101,6 +101,7 @@ class MixViewSet(viewsets.ModelViewSet):
'slug',
'user__slug',
'is_featured',
'genres__slug',
)
ordering_fields = (
@@ -121,7 +122,7 @@ class MixViewSet(viewsets.ModelViewSet):
raise PermissionDenied("Not allowed")
if 'random' in self.request.query_params:
return self.queryset.order_by('?').all()
if 'slug' or 'user__slug' in self.kwargs:
if 'slug' in self.kwargs or 'user__slug' in self.kwargs:
""" could be private mix so don't filter """
return self.queryset
else:
@@ -181,6 +182,7 @@ class SearchResultsView(views.APIView):
'image': mix.get_image_url(size='48x48'),
'user': mix.user.slug,
'slug': mix.slug,
'user': mix.user.slug,
'url': mix.get_absolute_url(),
'description': mix.description
} for mix in Mix.objects.filter(title__icontains=q)[0:10]]

View File

@@ -7,10 +7,13 @@ logger = logging.getLogger('dss')
def post_activity(channel, message, session=''):
r = redis.StrictRedis(host=settings.REDIS_HOST, port=6379, db=0)
payload = json.dumps({'session': session, 'message': message})
response = r.publish(channel, payload)
logger.debug("Message sent: {0}".format(payload))
try:
r = redis.StrictRedis(host=settings.REDIS_HOST, port=6379, db=0)
payload = json.dumps({'session': session, 'message': message})
response = r.publish(channel, payload)
logger.debug("Message sent: {0}".format(payload))
except Exception as ex:
logger.error(ex)
if __name__ == '__main__':
post_activity('site:broadcast', 'bargle', '3a596ca6c97065a67aca3dc4a3ba230d688cf413')

View File

@@ -26,7 +26,7 @@ def set_azure_details(blob_name, download_name, container_name=AZURE_CONTAINER):
blob_service.set_blob_properties(
container_name,
blob_name,
x_ms_blob_content_type='application/octet-stream',
x_ms_blob_content_type='audio/mpeg',
x_ms_blob_content_disposition='attachment;filename="{0}"'.format(download_name)
)
print("Processed: %s" % download_name)
@@ -61,4 +61,4 @@ def enumerate_objects(container):
def delete_object(container, name):
blob_service = BlobService(AZURE_ACCOUNT_NAME, AZURE_ACCOUNT_KEY)
blob_service.delete_blob(container, name)
blob_service.delete_blob(container, name)

View File

@@ -1,5 +1,6 @@
import urllib.parse
import re
import urllib
from django.contrib.sites.models import Site
from django.template.defaultfilters import slugify

View File

@@ -3,7 +3,13 @@ import sys
from dss import localsettings
if os.name == 'posix':
LOG_FILE = localsettings.DSS_TEMP_PATH + '/dss.log'
LOG_FILE = localsettings.DSS_TEMP_PATH + 'dss.log'
if not os.path.exists(LOG_FILE):
print("Creating {}".format(LOG_FILE))
open(LOG_FILE, 'a').close()
else:
print("{} already exists.".format(LOG_FILE))
else:
LOG_FILE = 'c:\\temp\\dss.log'

View File

@@ -20,7 +20,7 @@ DEVELOPMENT = DEBUG
# AUTH_USER_MODEL = 'spa.UserProfile'
TEMPLATE_DEBUG = DEBUG
VERSION = '2.13.04'
VERSION = '3.0.1'
ADMINS = (
('Fergal Moran', 'fergal.moran@gmail.com'),
@@ -212,7 +212,7 @@ THUMBNAIL_PREFIX = '_tn/'
# THUMBNAIL_STORAGE = 'storages.backends.azure_storage.AzureStorage'
JWT_AUTH = {
'JWT_EXPIRATION_DELTA': timedelta(seconds=900),
'JWT_EXPIRATION_DELTA': timedelta(seconds=(60 * 60 * 24) * 14),
# 'JWT_EXPIRATION_DELTA': timedelta(seconds=5),
'JWT_ALLOW_REFRESH': True,
'JWT_REFRESH_EXPIRATION_DELTA': timedelta(days=30),
@@ -244,3 +244,12 @@ CORS_ALLOW_HEADERS = (
)
""" End static settings """
SOCIAL_AUTH_AUTHENTICATION_BACKENDS = (
'social.backends.open_id.OpenIdAuth',
'social.backends.google.GoogleOpenId',
'social.backends.google.GoogleOAuth2',
'social.backends.google.GoogleOAuth',
'social.backends.twitter.TwitterOAuth',
'social.backends.yahoo.YahooOpenId'
)

View File

@@ -10,47 +10,8 @@ from dropbox.rest import ErrorResponse
from dss import settings
""" Monkey patch dropbox upload chunked """
def __upload_chunked(self, chunk_size = 4 * 1024 * 1024):
"""Uploads data from this ChunkedUploader's file_obj in chunks, until
an error occurs. Throws an exception when an error occurs, and can
be called again to resume the upload.
Parameters
chunk_size
The number of bytes to put in each chunk. (Default 4 MB.)
"""
while self.offset < self.target_length:
next_chunk_size = min(chunk_size, self.target_length - self.offset)
if self.last_block == None:
self.last_block = self.file_obj.read(next_chunk_size)
try:
(self.offset, self.upload_id) = self.client.upload_chunk(
self.last_block, next_chunk_size, self.offset, self.upload_id)
self.last_block = None
except ErrorResponse as e:
# Handle the case where the server tells us our offset is wrong.
must_reraise = True
if e.status == 400:
reply = e.body
if "offset" in reply and reply['offset'] != 0 and reply['offset'] > self.offset:
self.last_block = None
self.offset = reply['offset']
must_reraise = False
if must_reraise:
raise
ChunkedUploader.upload_chunked = __upload_chunked
from dropbox.client import ChunkedUploader
""" Monkey patch dropbox upload chunked """
def __upload_chunked(self, chunk_size = 4 * 1024 * 1024):
"""Uploads data from this ChunkedUploader's file_obj in chunks, until
an error occurs. Throws an exception when an error occurs, and can
@@ -122,7 +83,8 @@ def _create_backup_bundle(remote_file, location):
backup_file = "{0}/{1}".format(settings.DSS_TEMP_PATH, remote_file)
tar = tarfile.open(backup_file, "w:gz")
tar.add(location)
print("Remote file: {}".format(remote_file.replace(".tar.gz", "")))
tar.add(tarfile.TarInfo(remote_file.replace(".tar.gz", "")), location)
tar.close()
return backup_file

View File

@@ -21,3 +21,4 @@ def download_file( url, file_name):
print(status, end=' ')
f.close()

View File

@@ -1,11 +1,50 @@
import os
import tarfile
from django.core.management.base import LabelCommand
import dropbox
from dss import settings
import sys
from utils import query_yes_no
def _restore_database():
""" find latest database backup """
client = dropbox.client.DropboxClient(settings.DSS_DB_BACKUP_TOKEN)
client = dropbox.Dropbox(settings.DSS_DB_BACKUP_TOKEN)
files = client.files_list_folder('/media')
latest = None
for f in files.entries:
print(f.server_modified)
if latest is None or f.server_modified > latest.server_modified:
latest = f
if latest is not None:
#if query_yes_no("Restoring backing from: {}\nProceed (y/n?)".format(latest)):
print("Restoring database")
backup_file = '/tmp/{}'.format(latest.name)
result = client.files_download_to_file(backup_file, latest.path_lower)
print("Downloaded {}".format(result))
if os.path.exists(backup_file):
print("Download completed")
o = tarfile.open(backup_file)
o.extract()
else:
print("Unable to download file")
"""
import requests
...
headers = ... # set up auth
...
params = { 'list' : 'true' }
response = requests.get('https://api.dropbox.com/1/metadata/dropbox/<directory>', params=params, headers=headers)
subdirs = [d['path'] for d in response.json()['contents'] if d['is_dir'] == True]
print(subdirs)
"""
class Command(LabelCommand):

View File

@@ -23,6 +23,6 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='mix',
name='is_private',
field=models.BooleanField(default=True),
field=models.BooleanField(default=False),
),
]

View File

@@ -96,35 +96,32 @@ class Mix(BaseModel):
return self.__unicode__()
def __unicode__(self):
return "{} - {}".format(self.user.get_nice_name(), self.title)
return "{} - {}".format(self.user.get_nice_name(), self.title)
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
if not self.id:
self.slug = unique_slugify(self, self.title)
self.clean_image('mix_image', Mix)
# Check for the unlikely event that the waveform has been generated
if cdn.file_exists('{0}{1}.png'.format(settings.WAVEFORM_URL, self.uid)):
self.waveform_generated = True
try:
self.duration = mp3_length(self.get_absolute_path())
except Mp3FileNotFoundException:
# Not really bothered about this in save as it can be called before we have an mp3
pass
super(Mix, self).save(force_insert, force_update, using, update_fields)
def set_cdn_details(self, path):
self.waveform_generated = True
self.duration = mp3_length(path)
self.save(update_fields=["waveform_generated", "duration"])
self.update_file_http_headers(self.uid, self.title)
def create_mp3_tags(self, prefix=""):
try:
tag_mp3(
self.get_absolute_path(),
artist=self.user.get_nice_name(),
title=self.title,
url=self.get_full_url(),
album="Deep South Sounds Mixes",
year=self.upload_date.year,
comment=self.description,
genres=self.get_nice_genres())
self.get_absolute_path(),
artist=self.user.get_nice_name(),
title=self.title,
url=self.get_full_url(),
album="Deep South Sounds Mixes",
year=self.upload_date.year,
comment=self.description,
genres=self.get_nice_genres())
except Exception as ex:
self.logger.exception("Mix: error creating tags: %s" % ex)
pass
@@ -176,7 +173,7 @@ class Mix(BaseModel):
ret = get_thumbnail(self.mix_image, size, crop='center')
return url.urlclean("%s/%s" % (settings.MEDIA_URL, ret.name))
else:
return self.user.get_sized_avatar_image(170, 170)
return self.user.get_sized_avatar_image(253, 157)
except Exception as ex:
pass

View File

@@ -174,6 +174,7 @@ class UserProfile(BaseModel):
def get_sized_avatar_image(self, width, height):
try:
#import ipdb; ipdb.set_trace()
image = self.get_avatar_image()
logger.debug("get_sized_avatar_image: %s".format(image))
sized = thumbnail.get_thumbnail(image, "%sx%s" % (width, height), crop="center")

View File

@@ -1,16 +1,13 @@
from django.contrib.auth.models import User
from django.contrib.sessions.models import Session
from django.core.exceptions import ObjectDoesNotExist
from django.db.models.signals import post_save, pre_save, m2m_changed
from django.dispatch import Signal, receiver
from django.contrib.auth.models import User
from core.realtime import activity
from core.utils.audio.mp3 import mp3_length
from spa.models import SocialAccountLink
from spa.models.activity import ActivityFollow
from spa.models.userprofile import UserProfile
from spa.models.mix import Mix
from spa.models.userprofile import UserProfile
waveform_generated_signal = Signal()
@@ -23,10 +20,7 @@ def _waveform_generated_callback(sender, **kwargs):
if uid is not None:
mix = Mix.objects.get(uid=uid)
if mix is not None:
mix.waveform_generated = True
mix.duration = mp3_length(path)
mix.save(update_fields=["waveform_generated", "duration"])
mix.set_cdn_details(path)
except ObjectDoesNotExist:
print("Mix has still not been uploaded")
pass

View File

@@ -36,6 +36,14 @@ def create_waveform_task(in_file, uid):
logger.error("Outfile is missing")
@task(time_limit=3600)
def update_file_http_headers(uid, title):
cdn.set_azure_details(
blob_name='{0}.mp3'.format(uid),
download_name='Deep South Sounds - {0}.mp3'.format(title),
container_name='mixes')
@task(time_limit=3600)
def upload_to_cdn_task(filetype, uid, container_name):
source_file = os.path.join(settings.CACHE_ROOT, '{0}/{1}.{2}'.format(container_name, uid, filetype))