[Branch,~linaro-validation/lava-dispatcher/trunk] Rev 329: updates for new downloader code and qemu

=== modified file 'lava_dispatcher/client/master.py'
--- lava_dispatcher/client/master.py	2012-06-13 07:35:23 +0000
+++ lava_dispatcher/client/master.py	2012-06-19 04:18:31 +0000
@@ -34,7 +34,7 @@ 
 import errno
 
 from lava_dispatcher.utils import (
-    download,
+    download_image,
     logging_spawn,
     logging_system,
     string_to_list,
@@ -354,18 +354,6 @@ 
     def _close_logging_spawn(self):
         self.proc.close(True)
 
-    def decompress(self, image_file):
-        for suffix, command in [('.gz', 'gunzip'),
-                                ('.xz', 'unxz'),
-                                ('.bz2', 'bunzip2')]:
-            if image_file.endswith(suffix):
-                logging.info("Uncompressing %s with %s", image_file, command)
-                uncompressed_name = image_file[:-len(suffix)]
-                subprocess.check_call(
-                    [command, '-c', image_file], stdout=open(uncompressed_name, 'w'))
-                return uncompressed_name
-        return image_file
-
     def _tarball_url_to_cache(self, url, cachedir):
         cache_loc = url_to_cache(url, cachedir)
         # can't have a folder name same as file name. replacing '.' with '.'
@@ -426,6 +414,14 @@ 
                 raise
         return True
 
+    def _remove_cache_lock(self, image, lava_cachedir, cache_loc=None):
+        if not cache_loc:
+            cache_loc = self._tarball_url_to_cache(image, lava_cachedir)
+        path = os.path.join(cache_loc, "tarballs-cache-ongoing")
+        if os.path.exists(path):
+            logging.debug("Removing cache lock for %s" % path)
+            shutil.rmtree(path)
+
     def _cache_tarballs(self, image, boot_tgz, root_tgz, lava_cachedir):
         cache_loc = self._tarball_url_to_cache(image, lava_cachedir)
         if not os.path.exists(cache_loc):
@@ -434,9 +430,6 @@ 
         c_root_tgz = os.path.join(cache_loc, "root.tgz")
         shutil.copy(boot_tgz, c_boot_tgz)
         shutil.copy(root_tgz, c_root_tgz)
-        path = os.path.join(cache_loc, "tarballs-cache-ongoing")
-        if os.path.exists(path):
-            shutil.rmtree(path)
 
     def _download(self, url, directory):
         lava_proxy = self.context.lava_proxy
@@ -476,15 +469,16 @@ 
                         # caching of same tarballs exact at the same time. One of them will successfully
                         # get the lock directory. The rest will skip the caching if _about_to_cache_tarballs
                         # return false.
-                        should_cache = self._about_to_cache_tarballs(image, lava_cachedir)
-                        image_file = self._download(image, tarball_dir)
-                        image_file = self.decompress(image_file)
-                        boot_tgz, root_tgz = self._generate_tarballs(image_file)
-                        if should_cache:
-                            self._cache_tarballs(image, boot_tgz, root_tgz, lava_cachedir)
+                        try:
+                            should_cache = self._about_to_cache_tarballs(image, lava_cachedir)
+                            image_file = download_image(image, self.context, tarball_dir)
+                            boot_tgz, root_tgz = self._generate_tarballs(image_file)
+                            if should_cache:
+                                self._cache_tarballs(image, boot_tgz, root_tgz, lava_cachedir)
+                        finally:
+                            self._remove_cache_lock(image, lava_cachedir)
                 else:
-                    image_file = self._download(image, tarball_dir)
-                    image_file = self.decompress(image_file)
+                    image_file = download_image(image, self.context, tarball_dir)
                     boot_tgz, root_tgz = self._generate_tarballs(image_file)
                     # remove the cached tarballs
                     cache_loc = self._tarball_url_to_cache(image, lava_cachedir)

=== modified file 'lava_dispatcher/client/qemu.py'
--- lava_dispatcher/client/qemu.py	2012-05-08 14:01:53 +0000
+++ lava_dispatcher/client/qemu.py	2012-06-13 20:46:12 +0000
@@ -33,6 +33,7 @@ 
     image_partition_mounted,
     )
 from lava_dispatcher.utils import (
+    download_image,
     logging_spawn,
     logging_system,
     )
@@ -49,7 +50,7 @@ 
         if image is None:
             image_file = generate_image(self, hwpack, rootfs, kernel_matrix, use_cache, rootfstype)
         else:
-            image_file = image
+            image_file = download_image(image, self.context)
         self._lava_image = image_file
         with image_partition_mounted(self._lava_image, self.root_part) as mntdir:
             logging_system('echo linaro > %s/etc/hostname' % mntdir)

=== added file 'lava_dispatcher/default-config/lava-dispatcher/device-types/qemu.conf'
--- lava_dispatcher/default-config/lava-dispatcher/device-types/qemu.conf	1970-01-01 00:00:00 +0000
+++ lava_dispatcher/default-config/lava-dispatcher/device-types/qemu.conf	2012-06-13 20:47:32 +0000
@@ -0,0 +1,6 @@ 
+# NOTE: you also need to set something like:
+#  DEFAULT_QEMU_BINARY = qemu-system-arm
+# in your lava-dispatcher.conf file
+qemu_machine_type = beaglexm
+qemu_drive_interface = sd
+client_type=qemu

=== modified file 'lava_dispatcher/utils.py'
--- lava_dispatcher/utils.py	2012-06-07 23:38:28 +0000
+++ lava_dispatcher/utils.py	2012-06-13 20:20:03 +0000
@@ -23,9 +23,11 @@ 
 import logging
 import os
 import shutil
+import subprocess
 import urllib2
 import urlparse
 from shlex import shlex
+from tempfile import mkdtemp
 
 import pexpect
 
@@ -54,6 +56,28 @@ 
         raise RuntimeError("Could not retrieve %s" % url)
     return filename
 
+def decompress(image_file):
+    for suffix, command in [('.gz', 'gunzip'),
+                            ('.xz', 'unxz'),
+                            ('.bz2', 'bunzip2')]:
+        if image_file.endswith(suffix):
+            logging.info("Uncompressing %s with %s", image_file, command)
+            uncompressed_name = image_file[:-len(suffix)]
+            subprocess.check_call(
+                [command, '-c', image_file], stdout=open(uncompressed_name, 'w'))
+            return uncompressed_name
+    return image_file
+
+def download_image(url, context, imgdir=None):
+    ''' common download function to be used by clients. This will download
+    and decompress the image using LMC_COOKIES and/or LMC_PROXY settings
+    '''
+    logging.info("Downloading image: %s" % url)
+    if not imgdir:
+        imgdir = mkdtemp(dir=context.lava_image_tmpdir)
+    img = download(url, imgdir, context.lava_proxy, context.lava_cookies)
+    return decompress(img)
+
 def link_or_copy_file(src, dest):
     try:
         dir = os.path.dirname(dest)