=== modified file 'lava_dispatcher/client/base.py'
--- lava_dispatcher/client/base.py	2012-07-17 20:02:17 +0000
+++ lava_dispatcher/client/base.py	2012-07-21 22:56:42 +0000
@@ -19,14 +19,19 @@
 # along
 # with this program; if not, see <http://www.gnu.org/licenses>.
 
+import atexit
 import commands
 import contextlib
+import logging
+import os
 import pexpect
+import shutil
 import sys
 import time
+import traceback
+
 from cStringIO import StringIO
-import traceback
-import logging
+from tempfile import mkdtemp
 
 
 class CommandRunner(object):
@@ -424,6 +429,15 @@
     def get_seriallog(self):
         return self.sio.getvalue()
 
+    def get_www_scratch_dir(self):
+        ''' returns a temporary directory available for downloads that's gets
+        deleted when the process exits '''
+
+        d = mkdtemp(dir=self.context.lava_image_tmpdir)
+        atexit.register(shutil.rmtree, d)
+        os.chmod(d, 0755)
+        return d
+
     # Android stuff
 
     def get_android_adb_interface(self):

=== modified file 'lava_dispatcher/client/lmc_utils.py'
--- lava_dispatcher/client/lmc_utils.py	2012-06-29 01:26:45 +0000
+++ lava_dispatcher/client/lmc_utils.py	2012-07-22 00:43:26 +0000
@@ -9,14 +9,14 @@
 import sys
 
 from lava_dispatcher.client.base import CriticalError
+from lava_dispatcher.downloader import (
+    download_image,
+    )
 from lava_dispatcher.utils import (
-    download,
     logging_system,
     )
 
-def refresh_hwpack(client, kernel_matrix, hwpack):
-    lava_proxy = client.context.lava_proxy
-    LAVA_IMAGE_TMPDIR = client.context.lava_image_tmpdir
+def _refresh_hwpack(client, tarball_dir, kernel_matrix, hwpack):
     logging.info("Deploying new kernel")
     new_kernel = kernel_matrix[0]
     deb_prefix = kernel_matrix[1]
@@ -26,17 +26,14 @@
         raise CriticalError("New kernel only support deb kernel package!")
 
     # download package to local
-    tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR)
-    os.chmod(tarball_dir, 0755)
-    kernel_path = download(new_kernel, tarball_dir, lava_proxy)
-    hwpack_path = download(hwpack, tarball_dir, lava_proxy)
+    kernel_path = download_image(new_kernel, client.context, outdir, decompress=False)
+    hwpack_path = download_image(hwpack, client.context, outdir, decompress=False)
 
     cmd = ("sudo linaro-hwpack-replace -t %s -p %s -r %s"
             % (hwpack_path, kernel_path, deb_prefix))
 
     rc, output = getstatusoutput(cmd)
     if rc:
-        shutil.rmtree(tarball_dir)
         raise RuntimeError("linaro-hwpack-replace failed: %s" % output)
 
     #fix it:l-h-r doesn't make a output option to specify the output hwpack,
@@ -51,13 +48,13 @@
             return new_hwpack_path
 
 
-def generate_image(client, hwpack_url, rootfs_url, kernel_matrix, rootfstype=None):
+def generate_image(client, hwpack_url, rootfs_url, kernel_matrix,
+                    outdir, rootfstype=None):
     """Generate image from a hwpack and rootfs url
 
     :param hwpack_url: url of the Linaro hwpack to download
     :param rootfs_url: url of the Linaro image to download
     """
-    lava_proxy = client.context.lava_proxy
     LAVA_IMAGE_TMPDIR = client.context.lava_image_tmpdir
     LAVA_IMAGE_URL = client.context.lava_image_url
     logging.info("preparing to deploy on %s" % client.hostname)
@@ -65,20 +62,18 @@
     logging.info("  rootfs: %s" % rootfs_url)
     if kernel_matrix:
         logging.info("  package: %s" % kernel_matrix[0])
-        hwpack_url = refresh_hwpack(kernel_matrix, hwpack_url)
+        hwpack_url = _refresh_hwpack(client, outdir, kernel_matrix, hwpack_url)
         #make new hwpack downloadable
         hwpack_url = hwpack_url.replace(LAVA_IMAGE_TMPDIR, '')
         hwpack_url = '/'.join(u.strip('/') for u in [
             LAVA_IMAGE_URL, hwpack_url])
         logging.info("  hwpack with new kernel: %s" % hwpack_url)
-    tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR)
-    os.chmod(tarball_dir, 0755)
-    #fix me: if url is not http-prefix, copy it to tarball_dir
+
     logging.info("Downloading the %s file" % hwpack_url)
-    hwpack_path = download(hwpack_url, tarball_dir, lava_proxy)
+    hwpack_path = download_image(hwpack_url, client.context, outdir, decompress=False)
 
     logging.info("Downloading the %s file" % rootfs_url)
-    rootfs_path = download(rootfs_url, tarball_dir, lava_proxy)
+    rootfs_path = download_image(rootfs_url, client.context, outdir, decompress=False)
 
     logging.info("linaro-media-create version information")
     cmd = "sudo linaro-media-create -v"
@@ -87,7 +82,7 @@
     metadata['target.linaro-media-create-version'] = output
     client.context.test_data.add_metadata(metadata)
 
-    image_file = os.path.join(tarball_dir, "lava.img")
+    image_file = os.path.join(outdir, "lava.img")
 
     logging.info("client.device_type = %s" %client.device_type)
 
@@ -98,13 +93,18 @@
         cmd += ' --rootfs ' + rootfstype
     logging.info("Executing the linaro-media-create command")
     logging.info(cmd)
-    try:
-        _run_linaro_media_create(cmd)
-    except:
-        shutil.rmtree(tarball_dir)
-        raise
+
+    _run_linaro_media_create(cmd)
     return image_file
 
+def generate_fastmodel_image(hwpack, rootfs, odir, size="2000M"):
+    cmd = ("flock /var/lock/lava-lmc.lck sudo linaro-media-create "
+           "--dev fastmodel --output-directory %s --image-size %s "
+           "--hwpack %s --binary %s --hwpack-force-yes" %
+            (odir, size, hwpack, rootfs) )
+    logging.info("Generating fastmodel image with: %s" % cmd)
+    _run_linaro_media_create(cmd)
+
 def generate_android_image(device, boot, data, system, ofile, size="2000M"):
     cmd = ("flock /var/lock/lava-lmc.lck sudo linaro-android-media-create "
            "--dev %s --image_file %s --image_size %s "
@@ -244,3 +244,4 @@
         state = next_state_names[match_id]
         if state is None:
             return
+

=== modified file 'lava_dispatcher/client/master.py'
--- lava_dispatcher/client/master.py	2012-07-20 22:18:48 +0000
+++ lava_dispatcher/client/master.py	2012-07-22 01:40:55 +0000
@@ -25,7 +25,6 @@
 import re
 import shutil
 import subprocess
-from tempfile import mkdtemp
 import time
 import traceback
 import atexit
@@ -37,7 +36,6 @@
     download_image,
     )
 from lava_dispatcher.utils import (
-    download,
     logging_spawn,
     logging_system,
     string_to_list,
@@ -472,11 +470,6 @@
         shutil.copy(boot_tgz, c_boot_tgz)
         shutil.copy(root_tgz, c_root_tgz)
 
-    def _download(self, url, directory):
-        lava_proxy = self.context.lava_proxy
-        lava_cookies = self.context.lava_cookies
-        return download(url, directory, lava_proxy, lava_cookies)
-
     def deploy_linaro(self, hwpack=None, rootfs=None, image=None,
                       kernel_matrix=None, rootfstype='ext3'):
         LAVA_IMAGE_TMPDIR = self.context.lava_image_tmpdir
@@ -493,12 +486,12 @@
 
         # generate image if needed
         try:
+            tarball_dir = self.get_www_scratch_dir()
             if image is None:
-                image_file = generate_image(self, hwpack, rootfs, kernel_matrix)
+                image_file = generate_image(self, hwpack, rootfs,
+                    kernel_matrix, tarball_dir)
                 boot_tgz, root_tgz = self._generate_tarballs(image_file)
             else:
-                tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR)
-                atexit.register(shutil.rmtree, tarball_dir)
                 os.chmod(tarball_dir, 0755)
                 lava_cachedir = self.context.lava_cachedir
                 if self.context.job_data.get('health_check', False):
@@ -621,7 +614,6 @@
                     self.sio.write(tb)
                     raise CriticalError("Android deployment failed")
         finally:
-            shutil.rmtree(self.tarball_dir)
             logging.info("Android image deployment exiting")
 
     def _download_tarballs(self, boot_url, system_url, data_url, pkg_url=None):
@@ -632,18 +624,14 @@
         :param data_url: url of the Linaro Android data tarball to download
         :param pkg_url: url of the custom kernel tarball to download
         """
-        lava_proxy = self.context.lava_proxy
-        LAVA_IMAGE_TMPDIR = self.context.lava_image_tmpdir
-        self.tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR)
-        tarball_dir = self.tarball_dir
-        os.chmod(tarball_dir, 0755)
+        tarball_dir = self.get_www_scratch_dir()
         logging.info("Downloading the image files")
 
-        boot_path = download(boot_url, tarball_dir, lava_proxy)
-        system_path = download(system_url, tarball_dir, lava_proxy)
-        data_path = download(data_url, tarball_dir, lava_proxy)
+        boot_path = download_image(boot_url, self.context, tarball_dir, decompress=False)
+        system_path = download_image(system_url, self.context, tarball_dir, decompress=False)
+        data_path = download_image(data_url, self.context, tarball_dir, decompress=False)
         if pkg_url:
-            pkg_path = download(pkg_url, tarball_dir, lava_proxy)
+            pkg_path = download_image(pkg_url, self.context, tarball_dir, decompress=False)
         else:
             pkg_path = None
         logging.info("Downloaded the image files")
@@ -740,8 +728,7 @@
                 time.sleep(3)
 
                 result_tarball = "http://%s/lava_results.tgz" % master_ip
-                tarball_dir = mkdtemp(dir=self.context.lava_image_tmpdir)
-                os.chmod(tarball_dir, 0755)
+                tarball_dir = self.get_www_scratch_dir()
 
                 # download test result with a retry mechanism
                 # set retry timeout to 5 mins
@@ -752,7 +739,8 @@
 
                 while True:
                     try:
-                        result_path = download(result_tarball, tarball_dir)
+                        result_path = download_image(result_tarball,
+                            self.context, tarball_dir, decompress=False)
                         return 'pass', '', result_path
                     except RuntimeError:
                         tries += 1

=== modified file 'lava_dispatcher/client/qemu.py'
--- lava_dispatcher/client/qemu.py	2012-06-29 01:26:45 +0000
+++ lava_dispatcher/client/qemu.py	2012-07-21 22:56:42 +0000
@@ -49,7 +49,9 @@
 
     def deploy_linaro(self, hwpack=None, rootfs=None, image=None, kernel_matrix=None, rootfstype='ext3'):
         if image is None:
-            image_file = generate_image(self, hwpack, rootfs, kernel_matrix, rootfstype)
+            odir = self.get_www_scratch_dir()
+            image_file = generate_image(self, hwpack, rootfs, kernel_matrix,
+                odir, rootfstype)
         else:
             image_file = download_image(image, self.context)
         self._lava_image = image_file

=== modified file 'lava_dispatcher/utils.py'
--- lava_dispatcher/utils.py	2012-07-09 23:10:24 +0000
+++ lava_dispatcher/utils.py	2012-07-22 01:42:10 +0000
@@ -24,37 +24,11 @@
 import os
 import sys
 import shutil
-import urllib2
 import urlparse
 from shlex import shlex
 
 import pexpect
 
-def download(url, path="", proxy=None, cookies=None, verbose_failure=1):
-    urlpath = urlparse.urlsplit(url).path
-    filename = os.path.basename(urlpath)
-    if path:
-        filename = os.path.join(path, filename)
-    fd = open(filename, "w")
-    try:
-        if proxy:
-            handlers = [urllib2.ProxyHandler({'http': '%s' % proxy})]
-        else:
-            handlers = []
-        opener = urllib2.build_opener(*handlers)
-        if cookies:
-            opener.addheaders.append(('Cookie', cookies))
-        response = opener.open(urllib2.quote(url, safe=":/"), timeout=30)
-        fd = open(filename, 'wb')
-        shutil.copyfileobj(response, fd, 0x10000)
-        fd.close()
-        response.close()
-    except:
-        if verbose_failure:
-            logging.exception("download '%s' failed" % url)
-        raise RuntimeError("Could not retrieve %s" % url)
-    return filename
-
 def link_or_copy_file(src, dest):
     try:
         dir = os.path.dirname(dest)

