[Branch,~linaro-validation/lava-dashboard-tool/trunk] Rev 150: Improve backup and restore

=== modified file 'lava_dashboard_tool/commands.py'
--- lava_dashboard_tool/commands.py	2011-06-23 11:24:34 +0000
+++ lava_dashboard_tool/commands.py	2011-06-27 16:04:39 +0000
@@ -33,6 +33,9 @@ 
 import urlparse
 import xmlrpclib
 
+import simplejson
+from linaro_json.extensions import datetime_extension
+
 from lava_tool.interface import Command
 from lava_tool.commands import ExperimentalCommandMixIn
 
@@ -650,17 +653,33 @@ 
     def invoke_remote(self):
         if not os.path.exists(self.args.BACKUP_DIR):
             os.mkdir(self.args.BACKUP_DIR)
-        for stream in self.server.streams():
-            print "Processing stream %s" % stream["pathname"]
-            stream_dir = os.path.join(self.args.BACKUP_DIR, urllib.quote_plus(stream["pathname"]))
-            if not os.path.exists(stream_dir):
-                os.mkdir(stream_dir)
-            for bundle in self.server.bundles(stream["pathname"]):
-                print " * Backing up bundle %s" % bundle["content_filename"]
+        for bundle_stream in self.server.streams():
+            print "Processing stream %s" % bundle_stream["pathname"]
+            bundle_stream_dir = os.path.join(self.args.BACKUP_DIR, urllib.quote_plus(bundle_stream["pathname"]))
+            if not os.path.exists(bundle_stream_dir):
+                os.mkdir(bundle_stream_dir)
+            with open(os.path.join(bundle_stream_dir, "metadata.json"), "wt") as stream:
+                simplejson.dump({
+                    "pathname": bundle_stream["pathname"],
+                    "name": bundle_stream["name"],
+                    "user": bundle_stream["user"],
+                    "group": bundle_stream["group"],
+                }, stream)
+            for bundle in self.server.bundles(bundle_stream["pathname"]):
+                print " * Backing up bundle %s" % bundle["content_sha1"]
                 data = self.server.get(bundle["content_sha1"])
-                bundle_pathname = os.path.join(stream_dir, urllib.quote_plus(data["content_filename"]))
-                with open(bundle_pathname, "wb") as stream:
+                bundle_pathname = os.path.join(bundle_stream_dir, bundle["content_sha1"]) 
+                # Note: we write bundles as binary data to preserve anything the user might have dumped on us
+                with open(bundle_pathname + ".json", "wb") as stream:
                     stream.write(data["content"])
+                with open(bundle_pathname + ".metadata.json", "wt") as stream:
+                    simplejson.dump({
+                        "uploaded_by": bundle["uploaded_by"],
+                        "uploaded_on": datetime_extension.to_json(bundle["uploaded_on"]),
+                        "content_filename": bundle["content_filename"],
+                        "content_sha1": bundle["content_sha1"],
+                        "content_size": bundle["content_size"],
+                    }, stream)
 
 
 class restore(XMLRPCCommand):
@@ -681,17 +700,31 @@ 
             if not os.path.isdir(filesystem_stream_pathname):
                 continue
             stream_pathname = urllib.unquote(stream_pathname_quoted)
+            if os.path.exists(os.path.join(filesystem_stream_pathname, "metadata.json")):
+                with open(os.path.join(filesystem_stream_pathname, "metadata.json"), "rt") as stream:
+                    stream_metadata = simplejson.load(stream)
+            else:
+                stream_metadata = {}
             print "Processing stream %s" % stream_pathname
-            self.server.make_stream(stream_pathname, "Restored from backup")
-            for content_filename_quoted in os.listdir(filesystem_stream_pathname):
-                filesystem_content_filename = os.path.join(filesystem_stream_pathname, content_filename_quoted)
+            try:
+                self.server.make_stream(stream_pathname, stream_metadata.get("name", "Restored from backup"))
+            except xmlrpclib.Fault as ex:
+                if ex.faultCode != 409:
+                    raise
+            for content_sha1 in [item[:-len(".json")] for item in os.listdir(filesystem_stream_pathname) if item.endswith(".json") and not item.endswith(".metadata.json") and item != "metadata.json"]:
+                filesystem_content_filename = os.path.join(filesystem_stream_pathname, content_sha1 + ".json")
                 if not os.path.isfile(filesystem_content_filename):
                     continue
-                content_filename = urllib.unquote(content_filename_quoted)
-                print " * Restoring bundle %s" % content_filename
+                with open(os.path.join(filesystem_stream_pathname, content_sha1) + ".metadata.json", "rt") as stream:
+                    bundle_metadata = simplejson.load(stream)
                 with open(filesystem_content_filename, "rb") as stream:
                     content = stream.read()
-                self.server.put(content, content_filename, stream_pathname)
+                print " * Restoring bundle %s" % content_sha1
+                try:
+                    self.server.put(content, bundle_metadata["content_filename"], stream_pathname)
+                except xmlrpclib.Fault as ex:
+                    if ex.faultCode != 409:
+                        raise
             
 
 class pull(ExperimentalCommandMixIn, XMLRPCCommand):

=== modified file 'setup.py'
--- setup.py	2011-06-23 11:23:24 +0000
+++ setup.py	2011-06-27 12:36:35 +0000
@@ -60,6 +60,7 @@ 
         "Topic :: Software Development :: Testing"],
     install_requires=[
         'lava-tool >= 0.1',
+        'linaro-json >= 2.0',
         'versiontools >= 1.3.1'],
     setup_requires=['versiontools >= 1.3.1'],
     tests_require=[],