=== modified file 'lava_dispatcher/client/base.py'
@@ -19,14 +19,19 @@
# along
# with this program; if not, see <http://www.gnu.org/licenses>.
+import atexit
import commands
import contextlib
+import logging
+import os
import pexpect
+import shutil
import sys
import time
+import traceback
+
from cStringIO import StringIO
-import traceback
-import logging
+from tempfile import mkdtemp
class CommandRunner(object):
@@ -424,6 +429,15 @@
def get_seriallog(self):
return self.sio.getvalue()
+ def get_www_scratch_dir(self):
+ ''' returns a temporary directory available for downloads that's gets
+ deleted when the process exits '''
+
+ d = mkdtemp(dir=self.context.lava_image_tmpdir)
+ atexit.register(shutil.rmtree, d)
+ os.chmod(d, 0755)
+ return d
+
# Android stuff
def get_android_adb_interface(self):
=== modified file 'lava_dispatcher/client/lmc_utils.py'
@@ -9,14 +9,14 @@
import sys
from lava_dispatcher.client.base import CriticalError
+from lava_dispatcher.downloader import (
+ download_image,
+ )
from lava_dispatcher.utils import (
- download,
logging_system,
)
-def refresh_hwpack(client, kernel_matrix, hwpack):
- lava_proxy = client.context.lava_proxy
- LAVA_IMAGE_TMPDIR = client.context.lava_image_tmpdir
+def _refresh_hwpack(client, tarball_dir, kernel_matrix, hwpack):
logging.info("Deploying new kernel")
new_kernel = kernel_matrix[0]
deb_prefix = kernel_matrix[1]
@@ -26,17 +26,14 @@
raise CriticalError("New kernel only support deb kernel package!")
# download package to local
- tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR)
- os.chmod(tarball_dir, 0755)
- kernel_path = download(new_kernel, tarball_dir, lava_proxy)
- hwpack_path = download(hwpack, tarball_dir, lava_proxy)
+ kernel_path = download_image(new_kernel, client.context, outdir, decompress=False)
+ hwpack_path = download_image(hwpack, client.context, outdir, decompress=False)
cmd = ("sudo linaro-hwpack-replace -t %s -p %s -r %s"
% (hwpack_path, kernel_path, deb_prefix))
rc, output = getstatusoutput(cmd)
if rc:
- shutil.rmtree(tarball_dir)
raise RuntimeError("linaro-hwpack-replace failed: %s" % output)
#fix it:l-h-r doesn't make a output option to specify the output hwpack,
@@ -51,13 +48,13 @@
return new_hwpack_path
-def generate_image(client, hwpack_url, rootfs_url, kernel_matrix, rootfstype=None):
+def generate_image(client, hwpack_url, rootfs_url, kernel_matrix,
+ outdir, rootfstype=None):
"""Generate image from a hwpack and rootfs url
:param hwpack_url: url of the Linaro hwpack to download
:param rootfs_url: url of the Linaro image to download
"""
- lava_proxy = client.context.lava_proxy
LAVA_IMAGE_TMPDIR = client.context.lava_image_tmpdir
LAVA_IMAGE_URL = client.context.lava_image_url
logging.info("preparing to deploy on %s" % client.hostname)
@@ -65,20 +62,18 @@
logging.info(" rootfs: %s" % rootfs_url)
if kernel_matrix:
logging.info(" package: %s" % kernel_matrix[0])
- hwpack_url = refresh_hwpack(kernel_matrix, hwpack_url)
+ hwpack_url = _refresh_hwpack(client, outdir, kernel_matrix, hwpack_url)
#make new hwpack downloadable
hwpack_url = hwpack_url.replace(LAVA_IMAGE_TMPDIR, '')
hwpack_url = '/'.join(u.strip('/') for u in [
LAVA_IMAGE_URL, hwpack_url])
logging.info(" hwpack with new kernel: %s" % hwpack_url)
- tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR)
- os.chmod(tarball_dir, 0755)
- #fix me: if url is not http-prefix, copy it to tarball_dir
+
logging.info("Downloading the %s file" % hwpack_url)
- hwpack_path = download(hwpack_url, tarball_dir, lava_proxy)
+ hwpack_path = download_image(hwpack_url, client.context, outdir, decompress=False)
logging.info("Downloading the %s file" % rootfs_url)
- rootfs_path = download(rootfs_url, tarball_dir, lava_proxy)
+ rootfs_path = download_image(rootfs_url, client.context, outdir, decompress=False)
logging.info("linaro-media-create version information")
cmd = "sudo linaro-media-create -v"
@@ -87,7 +82,7 @@
metadata['target.linaro-media-create-version'] = output
client.context.test_data.add_metadata(metadata)
- image_file = os.path.join(tarball_dir, "lava.img")
+ image_file = os.path.join(outdir, "lava.img")
logging.info("client.device_type = %s" %client.device_type)
@@ -98,13 +93,18 @@
cmd += ' --rootfs ' + rootfstype
logging.info("Executing the linaro-media-create command")
logging.info(cmd)
- try:
- _run_linaro_media_create(cmd)
- except:
- shutil.rmtree(tarball_dir)
- raise
+
+ _run_linaro_media_create(cmd)
return image_file
+def generate_fastmodel_image(hwpack, rootfs, odir, size="2000M"):
+ cmd = ("flock /var/lock/lava-lmc.lck sudo linaro-media-create "
+ "--dev fastmodel --output-directory %s --image-size %s "
+ "--hwpack %s --binary %s --hwpack-force-yes" %
+ (odir, size, hwpack, rootfs) )
+ logging.info("Generating fastmodel image with: %s" % cmd)
+ _run_linaro_media_create(cmd)
+
def generate_android_image(device, boot, data, system, ofile, size="2000M"):
cmd = ("flock /var/lock/lava-lmc.lck sudo linaro-android-media-create "
"--dev %s --image_file %s --image_size %s "
@@ -244,3 +244,4 @@
state = next_state_names[match_id]
if state is None:
return
+
=== modified file 'lava_dispatcher/client/master.py'
@@ -25,7 +25,6 @@
import re
import shutil
import subprocess
-from tempfile import mkdtemp
import time
import traceback
import atexit
@@ -37,7 +36,6 @@
download_image,
)
from lava_dispatcher.utils import (
- download,
logging_spawn,
logging_system,
string_to_list,
@@ -472,11 +470,6 @@
shutil.copy(boot_tgz, c_boot_tgz)
shutil.copy(root_tgz, c_root_tgz)
- def _download(self, url, directory):
- lava_proxy = self.context.lava_proxy
- lava_cookies = self.context.lava_cookies
- return download(url, directory, lava_proxy, lava_cookies)
-
def deploy_linaro(self, hwpack=None, rootfs=None, image=None,
kernel_matrix=None, rootfstype='ext3'):
LAVA_IMAGE_TMPDIR = self.context.lava_image_tmpdir
@@ -493,12 +486,12 @@
# generate image if needed
try:
+ tarball_dir = self.get_www_scratch_dir()
if image is None:
- image_file = generate_image(self, hwpack, rootfs, kernel_matrix)
+ image_file = generate_image(self, hwpack, rootfs,
+ kernel_matrix, tarball_dir)
boot_tgz, root_tgz = self._generate_tarballs(image_file)
else:
- tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR)
- atexit.register(shutil.rmtree, tarball_dir)
os.chmod(tarball_dir, 0755)
lava_cachedir = self.context.lava_cachedir
if self.context.job_data.get('health_check', False):
@@ -621,7 +614,6 @@
self.sio.write(tb)
raise CriticalError("Android deployment failed")
finally:
- shutil.rmtree(self.tarball_dir)
logging.info("Android image deployment exiting")
def _download_tarballs(self, boot_url, system_url, data_url, pkg_url=None):
@@ -632,18 +624,14 @@
:param data_url: url of the Linaro Android data tarball to download
:param pkg_url: url of the custom kernel tarball to download
"""
- lava_proxy = self.context.lava_proxy
- LAVA_IMAGE_TMPDIR = self.context.lava_image_tmpdir
- self.tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR)
- tarball_dir = self.tarball_dir
- os.chmod(tarball_dir, 0755)
+ tarball_dir = self.get_www_scratch_dir()
logging.info("Downloading the image files")
- boot_path = download(boot_url, tarball_dir, lava_proxy)
- system_path = download(system_url, tarball_dir, lava_proxy)
- data_path = download(data_url, tarball_dir, lava_proxy)
+ boot_path = download_image(boot_url, self.context, tarball_dir, decompress=False)
+ system_path = download_image(system_url, self.context, tarball_dir, decompress=False)
+ data_path = download_image(data_url, self.context, tarball_dir, decompress=False)
if pkg_url:
- pkg_path = download(pkg_url, tarball_dir, lava_proxy)
+ pkg_path = download_image(pkg_url, self.context, tarball_dir, decompress=False)
else:
pkg_path = None
logging.info("Downloaded the image files")
@@ -740,8 +728,7 @@
time.sleep(3)
result_tarball = "http://%s/lava_results.tgz" % master_ip
- tarball_dir = mkdtemp(dir=self.context.lava_image_tmpdir)
- os.chmod(tarball_dir, 0755)
+ tarball_dir = self.get_www_scratch_dir()
# download test result with a retry mechanism
# set retry timeout to 5 mins
@@ -752,7 +739,8 @@
while True:
try:
- result_path = download(result_tarball, tarball_dir)
+ result_path = download_image(result_tarball,
+ self.context, tarball_dir, decompress=False)
return 'pass', '', result_path
except RuntimeError:
tries += 1
=== modified file 'lava_dispatcher/client/qemu.py'
@@ -49,7 +49,9 @@
def deploy_linaro(self, hwpack=None, rootfs=None, image=None, kernel_matrix=None, rootfstype='ext3'):
if image is None:
- image_file = generate_image(self, hwpack, rootfs, kernel_matrix, rootfstype)
+ odir = self.get_www_scratch_dir()
+ image_file = generate_image(self, hwpack, rootfs, kernel_matrix,
+ odir, rootfstype)
else:
image_file = download_image(image, self.context)
self._lava_image = image_file
=== modified file 'lava_dispatcher/utils.py'
@@ -24,37 +24,11 @@
import os
import sys
import shutil
-import urllib2
import urlparse
from shlex import shlex
import pexpect
-def download(url, path="", proxy=None, cookies=None, verbose_failure=1):
- urlpath = urlparse.urlsplit(url).path
- filename = os.path.basename(urlpath)
- if path:
- filename = os.path.join(path, filename)
- fd = open(filename, "w")
- try:
- if proxy:
- handlers = [urllib2.ProxyHandler({'http': '%s' % proxy})]
- else:
- handlers = []
- opener = urllib2.build_opener(*handlers)
- if cookies:
- opener.addheaders.append(('Cookie', cookies))
- response = opener.open(urllib2.quote(url, safe=":/"), timeout=30)
- fd = open(filename, 'wb')
- shutil.copyfileobj(response, fd, 0x10000)
- fd.close()
- response.close()
- except:
- if verbose_failure:
- logging.exception("download '%s' failed" % url)
- raise RuntimeError("Could not retrieve %s" % url)
- return filename
-
def link_or_copy_file(src, dest):
try:
dir = os.path.dirname(dest)