From patchwork Mon Nov 14 05:08:12 2011 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael-Doyle Hudson X-Patchwork-Id: 5090 Return-Path: X-Original-To: patchwork@peony.canonical.com Delivered-To: patchwork@peony.canonical.com Received: from fiordland.canonical.com (fiordland.canonical.com [91.189.94.145]) by peony.canonical.com (Postfix) with ESMTP id D8E3D23E0F for ; Mon, 14 Nov 2011 05:08:16 +0000 (UTC) Received: from mail-fx0-f52.google.com (mail-fx0-f52.google.com [209.85.161.52]) by fiordland.canonical.com (Postfix) with ESMTP id BB4FCA18060 for ; Mon, 14 Nov 2011 05:08:16 +0000 (UTC) Received: by faaa26 with SMTP id a26so113219faa.11 for ; Sun, 13 Nov 2011 21:08:16 -0800 (PST) Received: by 10.152.144.136 with SMTP id sm8mr13035729lab.33.1321247295840; Sun, 13 Nov 2011 21:08:15 -0800 (PST) X-Forwarded-To: linaro-patchwork@canonical.com X-Forwarded-For: patch@linaro.org linaro-patchwork@canonical.com Delivered-To: patches@linaro.org Received: by 10.152.40.7 with SMTP id t7cs27411lak; Sun, 13 Nov 2011 21:08:15 -0800 (PST) Received: by 10.181.13.82 with SMTP id ew18mr23515832wid.16.1321247293582; Sun, 13 Nov 2011 21:08:13 -0800 (PST) Received: from indium.canonical.com (indium.canonical.com. [91.189.90.7]) by mx.google.com with ESMTPS id ey3si9869014wbb.97.2011.11.13.21.08.13 (version=TLSv1/SSLv3 cipher=OTHER); Sun, 13 Nov 2011 21:08:13 -0800 (PST) Received-SPF: pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) client-ip=91.189.90.7; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) smtp.mail=bounces@canonical.com Received: from ackee.canonical.com ([91.189.89.26]) by indium.canonical.com with esmtp (Exim 4.71 #1 (Debian)) id 1RPomC-0003ns-Ua for ; Mon, 14 Nov 2011 05:08:12 +0000 Received: from ackee.canonical.com (localhost [127.0.0.1]) by ackee.canonical.com (Postfix) with ESMTP id DC976E01C4 for ; Mon, 14 Nov 2011 05:08:12 +0000 (UTC) MIME-Version: 1.0 X-Launchpad-Project: lava-dispatcher X-Launchpad-Branch: ~linaro-validation/lava-dispatcher/trunk X-Launchpad-Message-Rationale: Subscriber X-Launchpad-Branch-Revision-Number: 155 X-Launchpad-Notification-Type: branch-revision To: Linaro Patch Tracker From: noreply@launchpad.net Subject: [Branch ~linaro-validation/lava-dispatcher/trunk] Rev 155: fix pypi installations issues Message-Id: <20111114050812.26183.32456.launchpad@ackee.canonical.com> Date: Mon, 14 Nov 2011 05:08:12 -0000 Reply-To: noreply@launchpad.net Sender: bounces@canonical.com Errors-To: bounces@canonical.com Precedence: bulk X-Generated-By: Launchpad (canonical.com); Revision="14277"; Instance="launchpad-lazr.conf" X-Launchpad-Hash: 7844caabae7556666c599a3a534a95a70d79e96c Merge authors: Michael Hudson-Doyle (mwhudson) Zygmunt Krynicki (zkrynicki) Related merge proposals: https://code.launchpad.net/~zkrynicki/lava-dispatcher/fix-pypi-install-issues/+merge/81995 proposed by: Zygmunt Krynicki (zkrynicki) ------------------------------------------------------------ revno: 155 [merge] committer: Michael Hudson-Doyle branch nick: trunk timestamp: Mon 2011-11-14 18:06:05 +1300 message: fix pypi installations issues (and as a bonus, split __init__.py into {context,job,test_data}.py) added: lava_dispatcher/context.py lava_dispatcher/job.py lava_dispatcher/test_data.py modified: lava-dispatch lava_dispatcher/__init__.py setup.py --- lp:lava-dispatcher https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk You are subscribed to branch lp:lava-dispatcher. To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk/+edit-subscription === modified file 'lava-dispatch' --- lava-dispatch 2011-09-23 02:53:30 +0000 +++ lava-dispatch 2011-11-14 00:52:45 +0000 @@ -24,7 +24,7 @@ import sys import logging.config -from lava_dispatcher import LavaTestJob +from lava_dispatcher.job import LavaTestJob from lava_dispatcher.config import get_config parser = optparse.OptionParser('%prog: lava-dispatch ') === modified file 'lava_dispatcher/__init__.py' --- lava_dispatcher/__init__.py 2011-10-27 06:12:13 +0000 +++ lava_dispatcher/__init__.py 2011-11-11 14:47:22 +0000 @@ -18,190 +18,4 @@ # along # with this program; if not, see . -from datetime import datetime -import json -import tempfile -import traceback -from uuid import uuid1 -import base64 -import pexpect -import logging - -from lava_dispatcher.actions import get_all_cmds -from lava_dispatcher.config import get_config, get_device_config -from lava_dispatcher.client import LavaClient, CriticalError, GeneralError - -__version__ = "0.3.4" - -class LavaTestJob(object): - def __init__(self, job_json, oob_file): - self.job_status = 'pass' - self.load_job_data(job_json) - dispatcher_config = get_config("lava-dispatcher") - self.context = LavaContext( - self.target, self.image_type, dispatcher_config, oob_file, - self.job_data) - - def load_job_data(self, job_json): - self.job_data = json.loads(job_json) - - @property - def target(self): - return self.job_data['target'] - - @property - def image_type(self): - return self.job_data.get('image_type') - - def run(self): - lava_commands = get_all_cmds() - - if self.job_data['actions'][-1]['command'].startswith("submit_results"): - submit_results = self.job_data['actions'].pop(-1) - else: - submit_results = None - - metadata = { - 'target.hostname': self.target, - } - - if 'device_type' in self.job_data: - metadata['target.device_type'] = self.job_data['device_type'] - self.context.test_data.add_metadata(metadata) - - try: - for cmd in self.job_data['actions']: - params = cmd.get('parameters', {}) - metadata = cmd.get('metadata', {}) - self.context.test_data.add_metadata(metadata) - action = lava_commands[cmd['command']](self.context) - try: - status = 'fail' - action.run(**params) - except CriticalError as err: - raise - except (pexpect.TIMEOUT, GeneralError) as err: - pass - except Exception as err: - raise - else: - status = 'pass' - finally: - err_msg = "" - logging.info("Action %s finished." % cmd['command']) - if status == 'fail': - err_msg = "Lava failed at action %s with error: %s\n" %\ - (cmd['command'], err) - if cmd['command'] == 'lava_test_run': - err_msg += "Lava failed on test: %s" %\ - params.get('test_name', "Unknown") - err_msg = err_msg + traceback.format_exc() - # output to both serial log and logfile - self.context.client.sio.write(err_msg) - else: - err_msg = "" - self.context.test_data.add_result( - action.test_name(**params), status, err_msg) - except: - #Capture all user-defined and non-user-defined critical errors - self.context.test_data.job_status='fail' - raise - finally: - if submit_results: - params = submit_results.get('parameters', {}) - action = lava_commands[submit_results['command']]( - self.context) - action.run(**params) - - -class LavaContext(object): - def __init__(self, target, image_type, dispatcher_config, oob_file, job_data): - self.config = dispatcher_config - self.job_data = job_data - device_config = get_device_config(target) - self._client = LavaClient(self, device_config) - self.test_data = LavaTestData() - self.oob_file = oob_file - self._host_result_dir = None - - @property - def client(self): - return self._client - - @property - def lava_server_ip(self): - return self.config.get("LAVA_SERVER_IP") - - @property - def lava_image_tmpdir(self): - return self.config.get("LAVA_IMAGE_TMPDIR") - - @property - def lava_image_url(self): - return self.config.get("LAVA_IMAGE_URL") - - @property - def host_result_dir(self): - if self._host_result_dir is None: - self._host_result_dir = tempfile.mkdtemp() - return self._host_result_dir - - @property - def lava_result_dir(self): - return self.config.get("LAVA_RESULT_DIR") - - @property - def lava_cachedir(self): - return self.config.get("LAVA_CACHEDIR") - - -class LavaTestData(object): - def __init__(self, test_id='lava'): - self.job_status = 'pass' - self.metadata = {} - self._test_run = { 'test_results':[], 'attachments':[] } - self._test_run['test_id'] = test_id - self._assign_date() - self._assign_uuid() - - def _assign_date(self): - TIMEFORMAT = '%Y-%m-%dT%H:%M:%SZ' - self._test_run['time_check_performed'] = False - self._test_run['analyzer_assigned_date'] = datetime.strftime( - datetime.now(), TIMEFORMAT) - - def _assign_uuid(self): - self._test_run['analyzer_assigned_uuid'] = str(uuid1()) - - def add_result(self, test_case_id, result, message=""): - result_data = { - 'test_case_id': test_case_id, - 'result': result, - 'message': message - } - self._test_run['test_results'].append(result_data) - - def add_attachment(self, attachment): - self._test_run['attachments'].append(attachment) - - def add_metadata(self, metadata): - self.metadata.update(metadata) - - def get_metadata(self): - return self.metadata - - def get_test_run(self): - self.add_result('job_complete', self.job_status) - return self._test_run - - def add_seriallog(self, serial_log): - """ - Add serial log to the "attachments" field, it aligns bundle 1.2 format - """ - serial_log_base64 = base64.b64encode(serial_log) - attachment = { - "pathname": "serial.log", - "mime_type": "text/plain", - "content": serial_log_base64 } - self.add_attachment(attachment) - +__version__ = (0, 3, 4, "final", 0) === added file 'lava_dispatcher/context.py' --- lava_dispatcher/context.py 1970-01-01 00:00:00 +0000 +++ lava_dispatcher/context.py 2011-11-14 00:52:45 +0000 @@ -0,0 +1,66 @@ +# Copyright (C) 2011 Linaro Limited +# +# Author: Paul Larson +# +# This file is part of LAVA Dispatcher. +# +# LAVA Dispatcher is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# LAVA Dispatcher is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along +# with this program; if not, see . + +import tempfile + +from lava_dispatcher.client import LavaClient +from lava_dispatcher.config import get_device_config +from lava_dispatcher.test_data import LavaTestData + + +class LavaContext(object): + def __init__(self, target, image_type, dispatcher_config, oob_file, job_data): + self.config = dispatcher_config + self.job_data = job_data + device_config = get_device_config(target) + self._client = LavaClient(self, device_config) + self.test_data = LavaTestData() + self.oob_file = oob_file + self._host_result_dir = None + + @property + def client(self): + return self._client + + @property + def lava_server_ip(self): + return self.config.get("LAVA_SERVER_IP") + + @property + def lava_image_tmpdir(self): + return self.config.get("LAVA_IMAGE_TMPDIR") + + @property + def lava_image_url(self): + return self.config.get("LAVA_IMAGE_URL") + + @property + def host_result_dir(self): + if self._host_result_dir is None: + self._host_result_dir = tempfile.mkdtemp() + return self._host_result_dir + + @property + def lava_result_dir(self): + return self.config.get("LAVA_RESULT_DIR") + + @property + def lava_cachedir(self): + return self.config.get("LAVA_CACHEDIR") === added file 'lava_dispatcher/job.py' --- lava_dispatcher/job.py 1970-01-01 00:00:00 +0000 +++ lava_dispatcher/job.py 2011-11-11 14:47:22 +0000 @@ -0,0 +1,110 @@ +# Copyright (C) 2011 Linaro Limited +# +# Author: Paul Larson +# +# This file is part of LAVA Dispatcher. +# +# LAVA Dispatcher is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# LAVA Dispatcher is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along +# with this program; if not, see . + +import json +import logging +import pexpect +import traceback + +from lava_dispatcher.actions import get_all_cmds +from lava_dispatcher.client import CriticalError, GeneralError +from lava_dispatcher.config import get_config +from lava_dispatcher.context import LavaContext + + +class LavaTestJob(object): + def __init__(self, job_json, oob_file): + self.job_status = 'pass' + self.load_job_data(job_json) + dispatcher_config = get_config("lava-dispatcher") + self.context = LavaContext( + self.target, self.image_type, dispatcher_config, oob_file, + self.job_data) + + def load_job_data(self, job_json): + self.job_data = json.loads(job_json) + + @property + def target(self): + return self.job_data['target'] + + @property + def image_type(self): + return self.job_data.get('image_type') + + def run(self): + lava_commands = get_all_cmds() + + if self.job_data['actions'][-1]['command'].startswith("submit_results"): + submit_results = self.job_data['actions'].pop(-1) + else: + submit_results = None + + metadata = { + 'target.hostname': self.target, + } + + if 'device_type' in self.job_data: + metadata['target.device_type'] = self.job_data['device_type'] + self.context.test_data.add_metadata(metadata) + + try: + for cmd in self.job_data['actions']: + params = cmd.get('parameters', {}) + metadata = cmd.get('metadata', {}) + self.context.test_data.add_metadata(metadata) + action = lava_commands[cmd['command']](self.context) + try: + status = 'fail' + action.run(**params) + except CriticalError as err: + raise + except (pexpect.TIMEOUT, GeneralError) as err: + pass + except Exception as err: + raise + else: + status = 'pass' + finally: + err_msg = "" + logging.info("Action %s finished." % cmd['command']) + if status == 'fail': + err_msg = "Lava failed at action %s with error: %s\n" %\ + (cmd['command'], err) + if cmd['command'] == 'lava_test_run': + err_msg += "Lava failed on test: %s" %\ + params.get('test_name', "Unknown") + err_msg = err_msg + traceback.format_exc() + # output to both serial log and logfile + self.context.client.sio.write(err_msg) + else: + err_msg = "" + self.context.test_data.add_result( + action.test_name(**params), status, err_msg) + except: + #Capture all user-defined and non-user-defined critical errors + self.context.test_data.job_status='fail' + raise + finally: + if submit_results: + params = submit_results.get('parameters', {}) + action = lava_commands[submit_results['command']]( + self.context) + action.run(**params) === added file 'lava_dispatcher/test_data.py' --- lava_dispatcher/test_data.py 1970-01-01 00:00:00 +0000 +++ lava_dispatcher/test_data.py 2011-11-11 14:47:22 +0000 @@ -0,0 +1,73 @@ +# Copyright (C) 2011 Linaro Limited +# +# Author: Paul Larson +# +# This file is part of LAVA Dispatcher. +# +# LAVA Dispatcher is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# LAVA Dispatcher is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along +# with this program; if not, see . + +from datetime import datetime +from uuid import uuid1 +import base64 + +class LavaTestData(object): + def __init__(self, test_id='lava'): + self.job_status = 'pass' + self.metadata = {} + self._test_run = { 'test_results':[], 'attachments':[] } + self._test_run['test_id'] = test_id + self._assign_date() + self._assign_uuid() + + def _assign_date(self): + TIMEFORMAT = '%Y-%m-%dT%H:%M:%SZ' + self._test_run['time_check_performed'] = False + self._test_run['analyzer_assigned_date'] = datetime.strftime( + datetime.now(), TIMEFORMAT) + + def _assign_uuid(self): + self._test_run['analyzer_assigned_uuid'] = str(uuid1()) + + def add_result(self, test_case_id, result, message=""): + result_data = { + 'test_case_id': test_case_id, + 'result': result, + 'message': message + } + self._test_run['test_results'].append(result_data) + + def add_attachment(self, attachment): + self._test_run['attachments'].append(attachment) + + def add_metadata(self, metadata): + self.metadata.update(metadata) + + def get_metadata(self): + return self.metadata + + def get_test_run(self): + self.add_result('job_complete', self.job_status) + return self._test_run + + def add_seriallog(self, serial_log): + """ + Add serial log to the "attachments" field, it aligns bundle 1.2 format + """ + serial_log_base64 = base64.b64encode(serial_log) + attachment = { + "pathname": "serial.log", + "mime_type": "text/plain", + "content": serial_log_base64 } + self.add_attachment(attachment) === modified file 'setup.py' --- setup.py 2011-09-09 00:52:17 +0000 +++ setup.py 2011-11-11 14:38:21 +0000 @@ -1,11 +1,10 @@ #!/usr/bin/env python from setuptools import setup, find_packages -from lava_dispatcher import __version__ as version setup( name="lava-dispatcher", - version=version, + version=":versiontools:lava_dispatcher:", url='https://launchpad.net/lava-dispatcher', license='GPL v2 or later', description="Part of the LAVA framework for dispatching test jobs", @@ -21,6 +20,12 @@ 'default-config/lava-dispatcher/devices/*.conf', ], }, + install_requires=[ + "pexpect >= 2.3", + ], + setup_requires=[ + 'versiontools >= 1.8', + ], scripts = [ 'lava-dispatch' ],