From patchwork Mon Jun 27 17:10:43 2011 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Zygmunt Krynicki X-Patchwork-Id: 2333 Return-Path: X-Original-To: patchwork@peony.canonical.com Delivered-To: patchwork@peony.canonical.com Received: from fiordland.canonical.com (fiordland.canonical.com [91.189.94.145]) by peony.canonical.com (Postfix) with ESMTP id 3D92323F70 for ; Mon, 27 Jun 2011 17:10:47 +0000 (UTC) Received: from mail-qy0-f180.google.com (mail-qy0-f180.google.com [209.85.216.180]) by fiordland.canonical.com (Postfix) with ESMTP id E4740A181FB for ; Mon, 27 Jun 2011 17:10:46 +0000 (UTC) Received: by qyk30 with SMTP id 30so3508541qyk.11 for ; Mon, 27 Jun 2011 10:10:46 -0700 (PDT) Received: by 10.229.102.98 with SMTP id f34mr1837545qco.42.1309194646223; Mon, 27 Jun 2011 10:10:46 -0700 (PDT) X-Forwarded-To: linaro-patchwork@canonical.com X-Forwarded-For: patch@linaro.org linaro-patchwork@canonical.com Delivered-To: patches@linaro.org Received: by 10.229.48.135 with SMTP id r7cs47899qcf; Mon, 27 Jun 2011 10:10:45 -0700 (PDT) Received: by 10.227.6.18 with SMTP id 18mr5948574wbx.66.1309194644758; Mon, 27 Jun 2011 10:10:44 -0700 (PDT) Received: from adelie.canonical.com (adelie.canonical.com [91.189.90.139]) by mx.google.com with ESMTP id fw18si11751669wbb.31.2011.06.27.10.10.44; Mon, 27 Jun 2011 10:10:44 -0700 (PDT) Received-SPF: pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.139 as permitted sender) client-ip=91.189.90.139; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.139 as permitted sender) smtp.mail=bounces@canonical.com Received: from loganberry.canonical.com ([91.189.90.37]) by adelie.canonical.com with esmtp (Exim 4.71 #1 (Debian)) id 1QbFKd-0003Ry-Tq for ; Mon, 27 Jun 2011 17:10:43 +0000 Received: from loganberry.canonical.com (localhost [127.0.0.1]) by loganberry.canonical.com (Postfix) with ESMTP id DB9502E800C for ; Mon, 27 Jun 2011 17:10:43 +0000 (UTC) MIME-Version: 1.0 X-Launchpad-Project: lava-dashboard-tool X-Launchpad-Branch: ~linaro-validation/lava-dashboard-tool/trunk X-Launchpad-Message-Rationale: Subscriber X-Launchpad-Branch-Revision-Number: 150 X-Launchpad-Notification-Type: branch-revision To: Linaro Patch Tracker From: noreply@launchpad.net Subject: [Branch ~linaro-validation/lava-dashboard-tool/trunk] Rev 150: Improve backup and restore Message-Id: <20110627171043.20999.79139.launchpad@loganberry.canonical.com> Date: Mon, 27 Jun 2011 17:10:43 -0000 Reply-To: noreply@launchpad.net Sender: bounces@canonical.com Errors-To: bounces@canonical.com Precedence: bulk X-Generated-By: Launchpad (canonical.com); Revision="13265"; Instance="initZopeless config overlay" X-Launchpad-Hash: 4adef51fe64e3d11204545a75aa25ebf0755fea4 Merge authors: Zygmunt Krynicki (zkrynicki) Related merge proposals: https://code.launchpad.net/~zkrynicki/lava-dashboard-tool/fix-and-improve-backup-restore/+merge/66015 proposed by: Zygmunt Krynicki (zkrynicki) review: Approve - Paul Larson (pwlars) ------------------------------------------------------------ revno: 150 [merge] fixes bug(s): https://launchpad.net/bugs/797622 https://launchpad.net/bugs/797628 committer: Zygmunt Krynicki branch nick: merge timestamp: Mon 2011-06-27 18:07:55 +0100 message: Improve backup and restore modified: lava_dashboard_tool/commands.py setup.py --- lp:lava-dashboard-tool https://code.launchpad.net/~linaro-validation/lava-dashboard-tool/trunk You are subscribed to branch lp:lava-dashboard-tool. To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-dashboard-tool/trunk/+edit-subscription === modified file 'lava_dashboard_tool/commands.py' --- lava_dashboard_tool/commands.py 2011-06-23 11:24:34 +0000 +++ lava_dashboard_tool/commands.py 2011-06-27 16:04:39 +0000 @@ -33,6 +33,9 @@ import urlparse import xmlrpclib +import simplejson +from linaro_json.extensions import datetime_extension + from lava_tool.interface import Command from lava_tool.commands import ExperimentalCommandMixIn @@ -650,17 +653,33 @@ def invoke_remote(self): if not os.path.exists(self.args.BACKUP_DIR): os.mkdir(self.args.BACKUP_DIR) - for stream in self.server.streams(): - print "Processing stream %s" % stream["pathname"] - stream_dir = os.path.join(self.args.BACKUP_DIR, urllib.quote_plus(stream["pathname"])) - if not os.path.exists(stream_dir): - os.mkdir(stream_dir) - for bundle in self.server.bundles(stream["pathname"]): - print " * Backing up bundle %s" % bundle["content_filename"] + for bundle_stream in self.server.streams(): + print "Processing stream %s" % bundle_stream["pathname"] + bundle_stream_dir = os.path.join(self.args.BACKUP_DIR, urllib.quote_plus(bundle_stream["pathname"])) + if not os.path.exists(bundle_stream_dir): + os.mkdir(bundle_stream_dir) + with open(os.path.join(bundle_stream_dir, "metadata.json"), "wt") as stream: + simplejson.dump({ + "pathname": bundle_stream["pathname"], + "name": bundle_stream["name"], + "user": bundle_stream["user"], + "group": bundle_stream["group"], + }, stream) + for bundle in self.server.bundles(bundle_stream["pathname"]): + print " * Backing up bundle %s" % bundle["content_sha1"] data = self.server.get(bundle["content_sha1"]) - bundle_pathname = os.path.join(stream_dir, urllib.quote_plus(data["content_filename"])) - with open(bundle_pathname, "wb") as stream: + bundle_pathname = os.path.join(bundle_stream_dir, bundle["content_sha1"]) + # Note: we write bundles as binary data to preserve anything the user might have dumped on us + with open(bundle_pathname + ".json", "wb") as stream: stream.write(data["content"]) + with open(bundle_pathname + ".metadata.json", "wt") as stream: + simplejson.dump({ + "uploaded_by": bundle["uploaded_by"], + "uploaded_on": datetime_extension.to_json(bundle["uploaded_on"]), + "content_filename": bundle["content_filename"], + "content_sha1": bundle["content_sha1"], + "content_size": bundle["content_size"], + }, stream) class restore(XMLRPCCommand): @@ -681,17 +700,31 @@ if not os.path.isdir(filesystem_stream_pathname): continue stream_pathname = urllib.unquote(stream_pathname_quoted) + if os.path.exists(os.path.join(filesystem_stream_pathname, "metadata.json")): + with open(os.path.join(filesystem_stream_pathname, "metadata.json"), "rt") as stream: + stream_metadata = simplejson.load(stream) + else: + stream_metadata = {} print "Processing stream %s" % stream_pathname - self.server.make_stream(stream_pathname, "Restored from backup") - for content_filename_quoted in os.listdir(filesystem_stream_pathname): - filesystem_content_filename = os.path.join(filesystem_stream_pathname, content_filename_quoted) + try: + self.server.make_stream(stream_pathname, stream_metadata.get("name", "Restored from backup")) + except xmlrpclib.Fault as ex: + if ex.faultCode != 409: + raise + for content_sha1 in [item[:-len(".json")] for item in os.listdir(filesystem_stream_pathname) if item.endswith(".json") and not item.endswith(".metadata.json") and item != "metadata.json"]: + filesystem_content_filename = os.path.join(filesystem_stream_pathname, content_sha1 + ".json") if not os.path.isfile(filesystem_content_filename): continue - content_filename = urllib.unquote(content_filename_quoted) - print " * Restoring bundle %s" % content_filename + with open(os.path.join(filesystem_stream_pathname, content_sha1) + ".metadata.json", "rt") as stream: + bundle_metadata = simplejson.load(stream) with open(filesystem_content_filename, "rb") as stream: content = stream.read() - self.server.put(content, content_filename, stream_pathname) + print " * Restoring bundle %s" % content_sha1 + try: + self.server.put(content, bundle_metadata["content_filename"], stream_pathname) + except xmlrpclib.Fault as ex: + if ex.faultCode != 409: + raise class pull(ExperimentalCommandMixIn, XMLRPCCommand): === modified file 'setup.py' --- setup.py 2011-06-23 11:23:24 +0000 +++ setup.py 2011-06-27 12:36:35 +0000 @@ -60,6 +60,7 @@ "Topic :: Software Development :: Testing"], install_requires=[ 'lava-tool >= 0.1', + 'linaro-json >= 2.0', 'versiontools >= 1.3.1'], setup_requires=['versiontools >= 1.3.1'], tests_require=[],