From patchwork Wed Nov 21 13:04:12 2012 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Senthil Kumaran X-Patchwork-Id: 13041 Return-Path: X-Original-To: patchwork@peony.canonical.com Delivered-To: patchwork@peony.canonical.com Received: from fiordland.canonical.com (fiordland.canonical.com [91.189.94.145]) by peony.canonical.com (Postfix) with ESMTP id CA36023FC2 for ; Wed, 21 Nov 2012 13:04:15 +0000 (UTC) Received: from mail-ie0-f180.google.com (mail-ie0-f180.google.com [209.85.223.180]) by fiordland.canonical.com (Postfix) with ESMTP id 590EFA196C1 for ; Wed, 21 Nov 2012 13:04:15 +0000 (UTC) Received: by mail-ie0-f180.google.com with SMTP id e10so9731955iej.11 for ; Wed, 21 Nov 2012 05:04:14 -0800 (PST) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20120113; h=x-forwarded-to:x-forwarded-for:delivered-to:received-spf :content-type:mime-version:x-launchpad-project:x-launchpad-branch :x-launchpad-message-rationale:x-launchpad-branch-revision-number :x-launchpad-notification-type:to:from:subject:message-id:date :reply-to:sender:errors-to:precedence:x-generated-by :x-launchpad-hash:x-gm-message-state; bh=xw0ZMnvjVXbUfUzkQvKWBreEw2f6ltnvDOwdEZso5J4=; b=GWBpx+9jxWZokpjW2ddMXMyfJiCAXMisMDwi3p8hXF9QzoZFZbqjhS8uZ1ucHNtq28 L4mywWKX0LkXOChOlarMe929kv7lDXMT6coA4TLETnl2kfNa1eJqFuOSNgkG1QsIytoY pcIupUd/Gy+sNsY8LJt9OM1hQlMeGbV0DIQRosXZifavd6+C6EAL60KnwJnSdVtn5lCe /NHNhK8diUO6cD1CAuO0aAuLaFi3SyDhL/3c5yz3Dnk8QrWUWJvWAk1lgapG6CQQjUB7 yfkZPz04AZ0JaxPMgVmlOajxa+dc0UXp5ToetF34yvvJcLVZlFOzn2YPlt+vy5/F11Hb LcMw== Received: by 10.50.213.69 with SMTP id nq5mr13348579igc.70.1353503054683; Wed, 21 Nov 2012 05:04:14 -0800 (PST) X-Forwarded-To: linaro-patchwork@canonical.com X-Forwarded-For: patch@linaro.org linaro-patchwork@canonical.com Delivered-To: patches@linaro.org Received: by 10.50.67.148 with SMTP id n20csp23293igt; Wed, 21 Nov 2012 05:04:14 -0800 (PST) Received: by 10.180.99.97 with SMTP id ep1mr19704784wib.12.1353503053434; Wed, 21 Nov 2012 05:04:13 -0800 (PST) Received: from indium.canonical.com (indium.canonical.com. [91.189.90.7]) by mx.google.com with ESMTPS id e57si15865weo.98.2012.11.21.05.04.12 (version=TLSv1/SSLv3 cipher=OTHER); Wed, 21 Nov 2012 05:04:13 -0800 (PST) Received-SPF: pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) client-ip=91.189.90.7; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) smtp.mail=bounces@canonical.com Received: from ackee.canonical.com ([91.189.89.26]) by indium.canonical.com with esmtp (Exim 4.71 #1 (Debian)) id 1Tb9yO-00087h-I4 for ; Wed, 21 Nov 2012 13:04:12 +0000 Received: from ackee.canonical.com (localhost [127.0.0.1]) by ackee.canonical.com (Postfix) with ESMTP id 794BDE0135 for ; Wed, 21 Nov 2012 13:04:12 +0000 (UTC) MIME-Version: 1.0 X-Launchpad-Project: lava-dispatcher X-Launchpad-Branch: ~linaro-validation/lava-dispatcher/trunk X-Launchpad-Message-Rationale: Subscriber X-Launchpad-Branch-Revision-Number: 460 X-Launchpad-Notification-Type: branch-revision To: Linaro Patch Tracker From: noreply@launchpad.net Subject: [Branch ~linaro-validation/lava-dispatcher/trunk] Rev 460: Add ability to pull a bzr/git repo that includes the test definition YAML Message-Id: <20121121130412.24478.71255.launchpad@ackee.canonical.com> Date: Wed, 21 Nov 2012 13:04:12 -0000 Reply-To: noreply@launchpad.net Sender: bounces@canonical.com Errors-To: bounces@canonical.com Precedence: bulk X-Generated-By: Launchpad (canonical.com); Revision="16293"; Instance="launchpad-lazr.conf" X-Launchpad-Hash: 264b36b68ad938d800a0a789865d66e0eafe9d6f X-Gm-Message-State: ALoCoQlbqjtixnU588l7hdnJuCaiSFovrfiUnq60XAZREtFxqdg2ihWzBeSQm8qkK8V5wGgZ6rlZ Merge authors: Senthil Kumaran S (stylesen) Related merge proposals: https://code.launchpad.net/~stylesen/lava-dispatcher/testdef-from-repo/+merge/134835 proposed by: Senthil Kumaran S (stylesen) review: Approve - Michael Hudson-Doyle (mwhudson) ------------------------------------------------------------ revno: 460 [merge] committer: Senthil Kumaran branch nick: trunk timestamp: Wed 2012-11-21 18:32:46 +0530 message: Add ability to pull a bzr/git repo that includes the test definition YAML files. modified: lava_dispatcher/actions/lava_test_shell.py --- lp:lava-dispatcher https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk You are subscribed to branch lp:lava-dispatcher. To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk/+edit-subscription === modified file 'lava_dispatcher/actions/lava_test_shell.py' --- lava_dispatcher/actions/lava_test_shell.py 2012-11-19 19:55:35 +0000 +++ lava_dispatcher/actions/lava_test_shell.py 2012-11-21 13:02:46 +0000 @@ -84,6 +84,8 @@ import json import yaml +import glob +import time import logging import os import pexpect @@ -166,17 +168,34 @@ parameters_schema = { 'type': 'object', 'properties': { - 'testdef_urls': {'type': 'array', 'items': {'type': 'string'}}, + 'testdef_urls': {'type': 'array', + 'items': {'type': 'string'}, + 'optional': True}, + 'testdef_repos': {'type': 'array', + 'items': {'type': 'object', + 'properties': + {'git-repo': {'type': 'string', + 'optional': True}, + 'bzr-repo': {'type': 'string', + 'optional': True}, + 'revision': {'type': 'string', + 'optional': True}, + 'testdef': {'type': 'string', + 'optional': True} + }, + 'additionalProperties': False}, + 'optional': True + }, 'timeout': {'type': 'integer', 'optional': True}, }, 'additionalProperties': False, } - def run(self, testdef_urls, timeout=-1): + def run(self, testdef_urls=None, testdef_repos=None, timeout=-1): target = self.client.target_device self._assert_target(target) - self._configure_target(target, testdef_urls) + self._configure_target(target, testdef_urls, testdef_repos) with target.runner() as runner: patterns = [ @@ -194,11 +213,74 @@ self._bundle_results(target) - def _get_test_definition(self, testdef_url, tmpdir): - testdef_file = download_image(testdef_url, self.context, tmpdir) - with open(testdef_file, 'r') as f: - logging.info('loading test definition') - return yaml.load(f) + def _get_test_definition(self, d, ldir, testdef_src, tmpdir, isrepo=False): + repo = None + test = 'lavatest.yaml' + + if isrepo: + if 'git-repo' in testdef_src: + repo = self._get_testdef_git_repo(testdef_src['git-repo'], + tmpdir, + testdef_src.get('revision')) + + if 'bzr-repo' in testdef_src: + repo = self._get_testdef_bzr_repo(testdef_src['bzr-repo'], + tmpdir, + testdef_src.get('revision')) + + if 'testdef' in testdef_src: + test = testdef_src['testdef'] + + with open(os.path.join(repo, test), 'r') as f: + testdef = yaml.load(f) + else: + test = download_image(testdef_src, self.context, tmpdir) + with open(test, 'r') as f: + testdef = yaml.load(f) + + logging.info('loaded test definition ...') + + # android mount the partition under /system, while ubuntu mounts under + # /, so we have hdir for where it is on the host and tdir for how the + # target will see the path + timestamp = str(time.time()) + hdir = '%s/tests/%s_%s' % (d, timestamp, testdef['metadata']['name']) + tdir = '%s/tests/%s_%s' % (ldir, timestamp, testdef['metadata']['name']) + self._copy_test(hdir, tdir, testdef, repo) + + return tdir + + def _get_testdef_git_repo(self, testdef_repo, tmpdir, revision): + cwd = os.getcwd() + gitdir = os.path.join(tmpdir, 'gittestrepo') + try: + subprocess.check_call(['git', 'clone', testdef_repo, gitdir]) + if revision: + os.chdir(gitdir) + subprocess.check_call(['git', 'checkout', revision]) + return gitdir + except Exception as e: + logging.error('Unable to get test definition from git\n' + str(e)) + finally: + os.chdir(cwd) + + def _get_testdef_bzr_repo(self, testdef_repo, tmpdir, revision): + bzrdir = os.path.join(tmpdir, 'bzrtestrepo') + try: + # As per bzr revisionspec, '-1' is "The last revision in a + # branch". + if revision is None: + revision = '-1' + + subprocess.check_call(['bzr', + 'branch', + '-r', + revision, + testdef_repo, + bzrdir]) + return bzrdir + except Exception as e: + logging.error('Unable to get test definition from bzr\n' + str(e)) def _copy_runner(self, mntdir, target): runner = target.deployment_data['lava_test_runner'] @@ -288,7 +370,7 @@ for cmd in testdef['install']['steps']: f.write('%s\n' % cmd) - def _copy_test(self, hostdir, targetdir, testdef): + def _copy_test(self, hostdir, targetdir, testdef, testdef_repo=None): self._sw_sources = [] utils.ensure_directory(hostdir) with open('%s/testdef.yaml' % hostdir, 'w') as f: @@ -306,11 +388,16 @@ for cmd in testdef['run']['steps']: f.write('%s\n' % cmd) + if testdef_repo: + for filepath in glob.glob(os.path.join(testdef_repo, '*')): + shutil.copy2(filepath, hostdir) + logging.info('copied all test files') + def _mk_runner_dirs(self, mntdir): utils.ensure_directory('%s/bin' % mntdir) utils.ensure_directory_empty('%s/tests' % mntdir) - def _configure_target(self, target, testdef_urls): + def _configure_target(self, target, testdef_urls, testdef_repos): ldir = target.deployment_data['lava_test_dir'] results_part = target.deployment_data['lava_test_results_part_attr'] @@ -320,15 +407,24 @@ self._mk_runner_dirs(d) self._copy_runner(d, target) testdirs = [] - for i, url in enumerate(testdef_urls): - testdef = self._get_test_definition(url, target.scratch_dir) - # android mount the partition under /system, while ubuntu - # mounts under /, so we have hdir for where it is on the host - # and tdir for how the target will see the path - hdir = '%s/tests/%d_%s' % (d, i, testdef.get('metadata').get('name')) - tdir = '%s/tests/%d_%s' % (ldir, i, testdef.get('metadata').get('name')) - self._copy_test(hdir, tdir, testdef) - testdirs.append(tdir) + + if testdef_urls: + for url in testdef_urls: + tdir = self._get_test_definition(d, + ldir, + url, + target.scratch_dir, + isrepo=False) + testdirs.append(tdir) + + if testdef_repos: + for repo in testdef_repos: + tdir = self._get_test_definition(d, + ldir, + repo, + target.scratch_dir, + isrepo=True) + testdirs.append(tdir) with open('%s/lava-test-runner.conf' % d, 'w') as f: for testdir in testdirs: