From 2b6ba4c11317fc2390f549ef0a5727ef9fabaf3d Mon Sep 17 00:00:00 2001 From: Fergal Moran Date: Sun, 8 Nov 2015 19:59:39 +0000 Subject: [PATCH 1/3] Settings --- .gitignore | 3 +-- spa/management/commands/backup.py | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index b68b21f..4cfa1d2 100644 --- a/.gitignore +++ b/.gitignore @@ -11,8 +11,7 @@ media/* build/* _working/* static/CACHE/* -dss/localsettings.py -dss/storagesettings.py +dss/devsettings.py dss/celery_settings.py dss.conf dss/debugsettings.py diff --git a/spa/management/commands/backup.py b/spa/management/commands/backup.py index b07c28a..03e8b2b 100644 --- a/spa/management/commands/backup.py +++ b/spa/management/commands/backup.py @@ -3,6 +3,8 @@ import subprocess from django.core.management.base import LabelCommand, CommandError from subprocess import Popen, PIPE, STDOUT import pexpect +from dropbox.client import ChunkedUploader + from dss import settings import tarfile import dropbox @@ -54,6 +56,7 @@ def _upload_to_dropbox(type, backup_file, remote_file): try: with open(backup_file, "rb") as f: client = dropbox.client.DropboxClient(settings.DSS_DB_BACKUP_TOKEN) + uploader = ChunkedUploader(client=client, file_obj=backup_file) response = client.put_file("{0}/{1}".format(type, remote_file), f, overwrite=True) os.remove(backup_file) From 3981ce288cf51460d97a5400d494f094a159e6aa Mon Sep 17 00:00:00 2001 From: Fergal Moran Date: Sun, 8 Nov 2015 20:17:59 +0000 Subject: [PATCH 2/3] Altered backup --- spa/management/commands/backup.py | 87 ++++++++++++++++++++++++------- 1 file changed, 68 insertions(+), 19 deletions(-) diff --git a/spa/management/commands/backup.py b/spa/management/commands/backup.py index 03e8b2b..f764b09 100644 --- a/spa/management/commands/backup.py +++ b/spa/management/commands/backup.py @@ -1,19 +1,55 @@ -from gzip import GzipFile -import subprocess -from django.core.management.base import LabelCommand, CommandError -from subprocess import Popen, PIPE, STDOUT +import os +import tarfile +import time + +import dropbox import pexpect -from dropbox.client import ChunkedUploader +from django.core.management.base import LabelCommand +from dropbox.client import ChunkedUploader, DropboxClient +from dropbox.rest import ErrorResponse from dss import settings -import tarfile -import dropbox -import os, time + +""" Monkey patch dropbox upload chunked """ + + +def __upload_chunked(self, chunk_size = 4 * 1024 * 1024): + """Uploads data from this ChunkedUploader's file_obj in chunks, until + an error occurs. Throws an exception when an error occurs, and can + be called again to resume the upload. + + Parameters + chunk_size + The number of bytes to put in each chunk. (Default 4 MB.) + """ + + while self.offset < self.target_length: + next_chunk_size = min(chunk_size, self.target_length - self.offset) + if self.last_block == None: + self.last_block = self.file_obj.read(next_chunk_size) + + try: + (self.offset, self.upload_id) = self.client.upload_chunk( + self.last_block, next_chunk_size, self.offset, self.upload_id) + self.last_block = None + except ErrorResponse as e: + # Handle the case where the server tells us our offset is wrong. + must_reraise = True + if e.status == 400: + reply = e.body + if "offset" in reply and reply['offset'] != 0 and reply['offset'] > self.offset: + self.last_block = None + self.offset = reply['offset'] + must_reraise = False + if must_reraise: + raise + +ChunkedUploader.upload_chunked = __upload_chunked def _backup_database(): print("Creating database backup") - file_name = "{0}.sql".format(time.strftime("%Y%m%d-%H%M%S")) + file_name = "{}.sql".format(time.strftime("%Y%m%d-%H%M%S")) backup_file = os.path.join(settings.DSS_TEMP_PATH, file_name) print('Backing up {} database to {}'.format(settings.DATABASE_NAME, file_name)) @@ -27,13 +63,16 @@ def _backup_database(): child.sendline(settings.DATABASE_PASSWORD) child.expect(pexpect.EOF, timeout=120) - _create_backup_bundle("{0}.tar.gz".format(file_name), 'database', backup_file) + zip_name = "{0}.tar.gz".format(file_name) + archive = _create_backup_bundle(zip_name, backup_file) + _upload_to_dropbox('database', archive, zip_name) def _backup_settings(): print("Creating settings backup") - file_name = "{0}.tar.gz".format(time.strftime("%Y%m%d-%H%M%S")) - _create_backup_bundle(file_name, 'settings', settings.PROJECT_ROOT) + zip_name = "{0}.tar.gz".format(time.strftime("%Y%m%d-%H%M%S")) + tar_file = _create_backup_bundle(zip_name, settings.PROJECT_ROOT) + _upload_to_dropbox('settings', tar_file, "{}.tar.gz".format(zip_name)) def _progress_filter(tarinfo): @@ -41,26 +80,23 @@ def _progress_filter(tarinfo): return tarinfo -def _create_backup_bundle(remote_file, type, location): +def _create_backup_bundle(remote_file, location): backup_file = "{0}/{1}".format(settings.DSS_TEMP_PATH, remote_file) tar = tarfile.open(backup_file, "w:gz") tar.add(location) tar.close() - - _upload_to_dropbox(type, backup_file, remote_file) + return backup_file def _upload_to_dropbox(type, backup_file, remote_file): print("Uploading {0} to dropbox".format(backup_file)) try: with open(backup_file, "rb") as f: - client = dropbox.client.DropboxClient(settings.DSS_DB_BACKUP_TOKEN) - uploader = ChunkedUploader(client=client, file_obj=backup_file) + client = dropbox.Dropbox(settings.DSS_DB_BACKUP_TOKEN) response = client.put_file("{0}/{1}".format(type, remote_file), f, overwrite=True) os.remove(backup_file) - print(response) except Exception as ex: print(ex) @@ -69,7 +105,20 @@ def _upload_to_dropbox(type, backup_file, remote_file): def _backup_media(): print("Creating media backup") file_name = "{0}.tar.gz".format(time.strftime("%Y%m%d-%H%M%S")) - _create_backup_bundle(file_name, 'media', settings.MEDIA_ROOT) + archive = _create_backup_bundle(file_name, settings.MEDIA_ROOT) + + size = os.path.getsize(archive) + upload_file = open(archive, 'rb') + + client = dropbox.client.DropboxClient(settings.DSS_DB_BACKUP_TOKEN) + uploader = client.get_chunked_uploader(upload_file, size) + while uploader.offset < size: + try: + upload = uploader.upload_chunked() + except Exception as e: + print("Error uploading: {0}".format(e)) + + uploader.finish('/media/{}'.format(file_name)) class Command(LabelCommand): From 73b028e7f0201af5329443966b94241a6a75959d Mon Sep 17 00:00:00 2001 From: Fergal Moran Date: Sun, 8 Nov 2015 20:22:26 +0000 Subject: [PATCH 3/3] Fixed chunked backup --- dss/localsettings.py | 55 +++++++++++++++++++++++++++++++ dss/storagesettings.py | 6 ++++ requirements.txt | 2 +- spa/management/commands/backup.py | 4 +-- 4 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 dss/localsettings.py create mode 100644 dss/storagesettings.py diff --git a/dss/localsettings.py b/dss/localsettings.py new file mode 100644 index 0000000..4191807 --- /dev/null +++ b/dss/localsettings.py @@ -0,0 +1,55 @@ +import os +from dss import devsettings + +DEBUG = True +DSS_TEMP_PATH = os.environ.get('DSS_TEMP_PATH', '/tmp/') +DSS_LAME_PATH = os.environ.get('DSS_LAME_PATH', '/usr/bin/sox') +DSS_WAVE_PATH = os.environ.get('DSS_WAVE_PATH', + '/home/fergalm/Dropbox/development/deepsouthsounds.com/dss.lib/wav2png/bin/Linux/wav2png') +GEOIP_PATH = os.environ.get('GEOIP_PATH', '/home/fergalm/Dropbox/Private/deepsouthsounds.com/working/geolite') + +DATABASE_PASSWORD = os.environ.get('DATABASE_PASSWORD', 'deepsouthsounds') +DATABASE_NAME = os.environ.get('DATABASE_NAME', 'deepsouthsounds') +DATABASE_USER = os.environ.get('DATABASE_USER', 'deepsouthsounds') +DATABASE_HOST = os.environ.get('DATABASE_HOST', 'localhost') + +STATIC_URL = '/assets/' +MEDIA_ROOT = os.environ.get('MEDIA_ROOT', '/mnt/dev/deepsouthsounds.com/media') +STATIC_ROOT = os.environ.get('STATIC_ROOT', '/home/fergalm/Dropbox/development/deepsouthsounds.com/cache/static') +CACHE_ROOT = os.environ.get('CACHE_ROOT', '/mnt/dev/deepsouthsounds.com/cache') + +MEDIA_URL = os.environ.get('MEDIA_URL', 'http://localhost/DSSMedia/') # '{0}media/'.format(CDN_URL) + +REDIS_HOST = os.environ.get('REDIS_HOST', 'localhost') +BROKER_URL = os.environ.get('BROKER_URL', 'amqp://guest:guest@localhost:5672//') +CELERY_ACCEPT_CONTENT = ['pickle', 'msgpack', 'json'] + +SECRET_KEY = os.environ.get('SECRET_KEY', devsettings.SECRET_KEY) +LIVE_ENABLED = os.environ.get('LIVE_ENABLED', False) + +ICE_HOST = os.environ.get('ICE_HOST', 'localhost') +ICE_MOUNT = os.environ.get('ICE_MOUNT =', 'dss') +ICE_PORT = os.environ.get('ICE_PORT', 8000) + +RADIO_HOST = os.environ.get('RADIO_HOST', 'localhost') +RADIO_PORT = os.environ.get('RADIO_PORT', 8888) + +MANDRILL_API_KEY = os.environ.get('MANDRILL_API_KEY', '') + +SOCIAL_AUTH_FACEBOOK_KEY = os.environ.get('SOCIAL_AUTH_FACEBOOK_KEY', devsettings.SOCIAL_AUTH_FACEBOOK_KEY) +SOCIAL_AUTH_FACEBOOK_SECRET = os.environ.get('SOCIAL_AUTH_FACEBOOK_SECRET', devsettings.SOCIAL_AUTH_FACEBOOK_SECRET) + +SOCIAL_AUTH_TWITTER_KEY = os.environ.get('SOCIAL_AUTH_TWITTER_KEY', devsettings.SOCIAL_AUTH_TWITTER_KEY) +SOCIAL_AUTH_TWITTER_SECRET = os.environ.get('SOCIAL_AUTH_TWITTER_SECRET', devsettings.SOCIAL_AUTH_TWITTER_SECRET) + +SOCIAL_AUTH_GOOGLE_OAUTH_KEY = os.environ.get('SOCIAL_AUTH_GOOGLE_OAUTH_KEY', devsettings.SOCIAL_AUTH_GOOGLE_OAUTH_KEY) +SOCIAL_AUTH_GOOGLE_OAUTH_SECRET = os.environ.get('SOCIAL_AUTH_GOOGLE_OAUTH_SECRET', devsettings.SOCIAL_AUTH_GOOGLE_OAUTH_SECRET) + +SOCIAL_AUTH_GOOGLE_PLUS_KEY = os.environ.get('SOCIAL_AUTH_GOOGLE_PLUS_KEY', devsettings.SOCIAL_AUTH_GOOGLE_PLUS_KEY) +SOCIAL_AUTH_GOOGLE_PLUS_SECRET = os.environ.get('SOCIAL_AUTH_GOOGLE_PLUS_SECRET', devsettings.SOCIAL_AUTH_GOOGLE_PLUS_SECRET) + +DSS_DB_BACKUP_KEY = os.environ.get('DSS_DB_BACKUP_KEY', devsettings.DSS_DB_BACKUP_KEY) +DSS_DB_BACKUP_SECRET = os.environ.get('DSS_DB_BACKUP_SECRET', devsettings.DSS_DB_BACKUP_SECRET) +DSS_DB_BACKUP_TOKEN = os.environ.get('DSS_DB_BACKUP_TOKEN', devsettings.DSS_DB_BACKUP_TOKEN) + +AZURE_ACCOUNT_KEY = os.environ.get('AZURE_ACCOUNT_KEY', devsettings.AZURE_ACCOUNT_KEY) diff --git a/dss/storagesettings.py b/dss/storagesettings.py new file mode 100644 index 0000000..2e30c6b --- /dev/null +++ b/dss/storagesettings.py @@ -0,0 +1,6 @@ +from dss import localsettings + +AZURE_ACCOUNT_NAME = 'dsscdn' +AZURE_CONTAINER = 'media' +AZURE_ACCOUNT_KEY = localsettings.AZURE_ACCOUNT_KEY +AZURE_ITEM_BASE_URL = 'https://dsscdn.blob.core.windows.net/' diff --git a/requirements.txt b/requirements.txt index ce09af2..2d69c2d 100755 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ django-sendfile Werkzeug psycopg2 gunicorn -dropbox==2.2.0 +dropbox django-dirtyfields django-storages django-user-sessions diff --git a/spa/management/commands/backup.py b/spa/management/commands/backup.py index f764b09..3fd4017 100644 --- a/spa/management/commands/backup.py +++ b/spa/management/commands/backup.py @@ -5,7 +5,7 @@ import time import dropbox import pexpect from django.core.management.base import LabelCommand -from dropbox.client import ChunkedUploader, DropboxClient +from dropbox.client import ChunkedUploader from dropbox.rest import ErrorResponse from dss import settings @@ -93,7 +93,7 @@ def _upload_to_dropbox(type, backup_file, remote_file): print("Uploading {0} to dropbox".format(backup_file)) try: with open(backup_file, "rb") as f: - client = dropbox.Dropbox(settings.DSS_DB_BACKUP_TOKEN) + client = dropbox.client.DropboxClient(settings.DSS_DB_BACKUP_TOKEN) response = client.put_file("{0}/{1}".format(type, remote_file), f, overwrite=True) os.remove(backup_file)