Added a backup tool to import/export store data

Author: marcjamot

api/restapi.py

@@ -1,4 +1,5 @@
 __author__ = 'chris'
+import backuptool
 import json
 import os
 from txrestapi.resource import APIResource
@@ -601,3 +602,29 @@ def upload_image(self, request):
             request.write(json.dumps({"success": False, "reason": e.message}, indent=4))
             request.finish()
             return server.NOT_DONE_YET
+
+    @POST('^/api/v1/backup_files')
+    def backup_files(self, request):
+        """Archives OpenBazaar files in a single tar archive."""
+        output = request.args["output"][0]
+        return backuptool.backupfiles(output)
+
+    @POST('^/api/v1/export_database')
+    def export_database(self, request):
+        """Exports given tables to the OpenBazaar folder."""
+        tables_and_columns = request.args["tables_and_columns"][0]
+        remove_previous = request.args["remove_previous"][0]
+        return backuptool.exportdatabase(tables_and_columns, remove_previous)
+
+    @POST('^/api/v1/restore_files')
+    def restore_files(self, request):
+        """Restores files of given archive to OpenBazaar folder."""
+        input_file = request.args["input_file"][0]
+        rem_db_files = request.args["rem_db_files"][0]
+        return backuptool.restorefiles(input_file, rem_db_files)
+
+    @POST('^/api/v1/import_database')
+    def import_database(self, request):
+        """Imports given CSV file to the database."""
+        remove_previous = request.args["remove_previous"][0]
+        return backuptool.importdatabase(remove_previous)

backuptool.py

@@ -0,0 +1,160 @@
+"""Import and export data for the OpenBazaar server."""
+__author__ = 'marc'
+from constants import DATA_FOLDER
+import csv
+import db.datastore as db
+import errno
+import os
+import re
+import shutil
+import sqlite3 as lite
+import tarfile
+import time
+
+_TABLES = {
+    'hashmap': ['hash', 'filepath'],
+    'profile': ['id', 'serializedUserInfo'],
+    'listings': ['id', 'serializedListings'],
+    'keys': ['type', 'privkey', 'pubkey'],
+    'followers': ['id', 'serializedFollowers'],
+    'following': ['id', 'serializedFollowing'],
+    'messages': [
+        'guid', 'handle', 'signed_pubkey', 'encryption_pubkey', 'subject',
+        'message_type', 'message', 'timestamp', 'avatar_hash', 'signature',
+        'outgoing'
+    ],
+    'notifications': ['guid', 'handle', 'message', 'timestamp', 'avatar_hash'],
+    'vendors': ['guid', 'ip', 'port', 'signedPubkey'],
+    'moderators': [
+        'guid', 'signedPubkey', 'encryptionKey', 'encryptionSignature',
+        'bitcoinKey', 'bitcoinSignature', 'handle'
+    ],
+    'purchases': [
+        'id', 'title', 'timestamp', 'btc', 'address', 'status', 'thumbnail',
+        'seller', 'proofSig'
+    ],
+    'sales': [
+        'id', 'title', 'timestamp', 'btc', 'address', 'status', 'thumbnail',
+        'seller'
+    ],
+    'dht': ['keyword', 'id', 'value', 'birthday']
+}
+
+def _getdatabase():
+    """Retrieves the OpenBazaar database file."""
+    database = db.Database()
+    return database.DATABASE
+
+def silentremove(filename):
+    """Silently removes a file if it exists."""
+    try:
+        os.remove(filename)
+    except OSError as err:
+        if err.errno != errno.ENOENT: # ENOENT: no such file or directory
+            raise
+
+def backupfiles(output=None):
+    """Archives OpenBazaar files in a single tar archive."""
+    os.chdir(DATA_FOLDER)
+
+    # Archive files
+    files = os.listdir(DATA_FOLDER)
+    if not output:
+        output = 'backup_{0}.tar.gz'.format(time.strftime('%Y-%m-%d'))
+    silentremove(output)
+    with tarfile.open(output, 'w:gz') as tar:
+        for fil in files:
+            tar.add(fil)
+        tar.close()
+    return True
+
+
+def restorefiles(input_file, remove_previous_database_files=False):
+    """Restores files of given archive to OpenBazaar folder."""
+    if not input_file:
+        return 'Input path is needed'
+    os.chdir(DATA_FOLDER)
+
+    # Remove existing database files if any
+    if remove_previous_database_files and os.path.exists('backup'):
+        shutil.rmtree('backup')
+
+    # Unarchive files
+    with tarfile.open(input_file, 'r:gz') as tar:
+        tar.extractall()
+
+    return True
+
+def _exportdatabase_tocsv(tables_and_columns):
+    """Reads the database for all given tables and stores them as CSV files."""
+    db_file = _getdatabase()
+    with lite.connect(db_file) as db_connection:
+        db_connection.text_factory = str
+        cursor = db_connection.cursor()
+        for table in tables_and_columns:
+            table_name = table[0]
+            table_columns = ', '.join(table[1])
+            query = "SELECT {0} FROM {1}".format(table_columns, table_name)
+            data = cursor.execute(query)
+            file_name = 'table_{0}.csv'.format(table_name)
+            file_path = os.path.join('backup', file_name)
+            with open(file_path, 'wb') as fil:
+                writer = csv.writer(fil)
+                writer.writerow(table[1])
+                writer.writerows(data)
+    return True
+
+def exportdatabase(table_list, remove_previous=False):
+    """Exports given tables to the OpenBazaar folder."""
+    os.chdir(DATA_FOLDER)
+
+    # Parse table list
+    table_list = table_list.replace(' ', '').split(',')
+    tables_and_columns = []
+    for table in table_list:
+        if table in _TABLES:
+            tables_and_columns.append((table, _TABLES[table]))
+        else:
+            return 'ERROR, Table not found: {0}'.format(table)
+
+    # Remove existing database files and re-make them
+    if remove_previous and os.path.exists('backup'):
+        shutil.rmtree('backup')
+    if not os.path.exists('backup'):
+        os.makedirs('backup')
+    return _exportdatabase_tocsv(tables_and_columns)
+
+def _importcsv_totable(file_name, delete_data_first=False):
+    """Imports given CSV file to the database."""
+    table_name = re.search(r'table_(\w+).csv', file_name).group(1)
+    db_file = _getdatabase()
+    with lite.connect(db_file) as db_connection:
+        db_connection.text_factory = str
+        cursor = db_connection.cursor()
+        if delete_data_first:
+            cursor.execute('DELETE FROM {0}'.format(table_name))
+        with open(file_name, 'rb') as fil:
+            reader = csv.reader(fil)
+            header = True
+            for row in reader:
+                if header:
+                    header = False
+                    columns = ', '.join(['?' for _ in row])
+                    insertsql = 'INSERT INTO {0} VALUES ({1})'.format(
+                        table_name, columns)
+                    rowlen = len(row)
+                else:
+                    if len(row) == rowlen:
+                        cursor.execute(insertsql, row)
+
+def importdatabase(delete_previous_data=False):
+    """Imports table files from the OpenBazaar folder."""
+    os.chdir(DATA_FOLDER)
+
+    # Restore database files to the database
+    result = True
+    if os.path.exists('backup'):
+        files = ['backup/{0}'.format(fil) for fil in os.listdir('backup')]
+        for fil in files:
+            result = result and _importcsv_totable(fil, delete_previous_data)
+    return result