From patchwork Thu Jul 21 02:47:15 2011 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Paul Larson X-Patchwork-Id: 2901 Return-Path: X-Original-To: patchwork@peony.canonical.com Delivered-To: patchwork@peony.canonical.com Received: from fiordland.canonical.com (fiordland.canonical.com [91.189.94.145]) by peony.canonical.com (Postfix) with ESMTP id EB6DA2405D for ; Thu, 21 Jul 2011 02:47:17 +0000 (UTC) Received: from mail-qy0-f173.google.com (mail-qy0-f173.google.com [209.85.216.173]) by fiordland.canonical.com (Postfix) with ESMTP id 96222A181A9 for ; Thu, 21 Jul 2011 02:47:17 +0000 (UTC) Received: by qyk10 with SMTP id 10so3714321qyk.11 for ; Wed, 20 Jul 2011 19:47:17 -0700 (PDT) Received: by 10.229.217.3 with SMTP id hk3mr8530209qcb.38.1311216437024; Wed, 20 Jul 2011 19:47:17 -0700 (PDT) X-Forwarded-To: linaro-patchwork@canonical.com X-Forwarded-For: patch@linaro.org linaro-patchwork@canonical.com Delivered-To: patches@linaro.org Received: by 10.229.217.78 with SMTP id hl14cs128744qcb; Wed, 20 Jul 2011 19:47:16 -0700 (PDT) Received: by 10.217.2.202 with SMTP id p52mr2170476wes.28.1311216435792; Wed, 20 Jul 2011 19:47:15 -0700 (PDT) Received: from adelie.canonical.com (adelie.canonical.com [91.189.90.139]) by mx.google.com with ESMTP id s44si1576816weq.83.2011.07.20.19.47.15; Wed, 20 Jul 2011 19:47:15 -0700 (PDT) Received-SPF: pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.139 as permitted sender) client-ip=91.189.90.139; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.139 as permitted sender) smtp.mail=bounces@canonical.com Received: from loganberry.canonical.com ([91.189.90.37]) by adelie.canonical.com with esmtp (Exim 4.71 #1 (Debian)) id 1QjjIB-0001yD-1b for ; Thu, 21 Jul 2011 02:47:15 +0000 Received: from loganberry.canonical.com (localhost [127.0.0.1]) by loganberry.canonical.com (Postfix) with ESMTP id 084452E8043 for ; Thu, 21 Jul 2011 02:47:15 +0000 (UTC) MIME-Version: 1.0 X-Launchpad-Project: lava-dispatcher X-Launchpad-Branch: ~linaro-validation/lava-dispatcher/trunk X-Launchpad-Message-Rationale: Subscriber X-Launchpad-Branch-Revision-Number: 77 X-Launchpad-Notification-Type: branch-revision To: Linaro Patch Tracker From: noreply@launchpad.net Subject: [Branch ~linaro-validation/lava-dispatcher/trunk] Rev 77: improve error handling Message-Id: <20110721024715.6703.59285.launchpad@loganberry.canonical.com> Date: Thu, 21 Jul 2011 02:47:15 -0000 Reply-To: noreply@launchpad.net Sender: bounces@canonical.com Errors-To: bounces@canonical.com Precedence: bulk X-Generated-By: Launchpad (canonical.com); Revision="13475"; Instance="initZopeless config overlay" X-Launchpad-Hash: 59d68156c5beb44ab44d8a630617ae5b5fd550a4 Merge authors: Spring Zhang (qzhang) Related merge proposals: https://code.launchpad.net/~qzhang/lava-dispatcher/simple-error-handler/+merge/64054 proposed by: Spring Zhang (qzhang) review: Needs Fixing - Paul Larson (pwlars) review: Resubmit - Spring Zhang (qzhang) ------------------------------------------------------------ revno: 77 [merge] committer: Paul Larson branch nick: lava-dispatcher timestamp: Wed 2011-07-20 19:13:21 -0500 message: improve error handling modified: lava_dispatcher/__init__.py lava_dispatcher/actions/android_0xbench.py lava_dispatcher/actions/android_deploy.py lava_dispatcher/actions/deploy.py lava_dispatcher/actions/lava-test.py lava_dispatcher/android_client.py lava_dispatcher/client.py --- lp:lava-dispatcher https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk You are subscribed to branch lp:lava-dispatcher. To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk/+edit-subscription === modified file 'lava_dispatcher/__init__.py' --- lava_dispatcher/__init__.py 2011-06-27 04:55:08 +0000 +++ lava_dispatcher/__init__.py 2011-07-20 07:30:07 +0000 @@ -18,13 +18,17 @@ # along # with this program; if not, see . +import sys from datetime import datetime import json +import traceback +from uuid import uuid1 +import base64 +import pexpect + from lava_dispatcher.actions import get_all_cmds -from lava_dispatcher.client import LavaClient +from lava_dispatcher.client import LavaClient, CriticalError, GeneralError from lava_dispatcher.android_client import LavaAndroidClient -from uuid import uuid1 -import base64 class LavaTestJob(object): def __init__(self, job_json): @@ -59,18 +63,41 @@ metadata['target.hostname'] = self.target self.context.test_data.add_metadata(metadata) action = lava_commands[cmd['command']](self.context) - action.run(**params) + try: + status = 'fail' + action.run(**params) + except CriticalError, err: + raise err + except (pexpect.TIMEOUT, GeneralError), err: + pass + except Exception, err: + raise + else: + status = 'pass' + finally: + if status == 'fail': + err_msg = "Lava failed at action " + cmd['command'] \ + + " with error: " + str(err) + "\n" + if cmd['command'] == 'lava_test_run': + err_msg = err_msg + "Lava failed with test: " \ + + test_name + exc_type, exc_value, exc_traceback = sys.exc_info() + err_msg = err_msg + repr(traceback.format_tb(exc_traceback)) + print >> sys.stderr, err_msg + else: + err_msg = "" + self.context.test_data.add_result(cmd['command'], + status, err_msg) except: - #FIXME: need to capture exceptions for later logging - #and try to continue from where we left off - self.context.test_data.job_status='fail' - raise + #Capture all user-defined and non-user-defined critical errors + self.context.test_data.job_status='fail' + raise finally: - if submit_results: - params = submit_results.get('parameters', {}) - action = lava_commands[submit_results['command']]( - self.context) - action.run(**params) + if submit_results: + params = submit_results.get('parameters', {}) + action = lava_commands[submit_results['command']]( + self.context) + action.run(**params) class LavaContext(object): @@ -112,8 +139,9 @@ def job_status(self, status): self._job_status = status - def add_result(self, test_case_id, result): - result_data = { 'test_case_id': test_case_id, 'result':result } + def add_result(self, test_case_id, result, message=""): + result_data = { 'test_case_id': test_case_id, 'result': result \ + , 'message': message} self._test_run['test_results'].append(result_data) def add_attachment(self, attachment): === modified file 'lava_dispatcher/actions/android_0xbench.py' --- lava_dispatcher/actions/android_0xbench.py 2011-06-27 04:55:08 +0000 +++ lava_dispatcher/actions/android_0xbench.py 2011-07-20 06:53:10 +0000 @@ -47,5 +47,6 @@ self.client.android_logcat_monitor(pattern, timeout = 1200) except pexpect.TIMEOUT: print "0xbench Test: TIMEOUT Fail" - - self.client.android_logcat_stop() + raise + finally: + self.client.android_logcat_stop() === modified file 'lava_dispatcher/actions/android_deploy.py' --- lava_dispatcher/actions/android_deploy.py 2011-07-06 17:16:22 +0000 +++ lava_dispatcher/actions/android_deploy.py 2011-07-20 10:08:14 +0000 @@ -25,6 +25,7 @@ import shutil from tempfile import mkdtemp from lava_dispatcher.utils import download, download_with_cache +from lava_dispatcher.client import CriticalError class cmd_deploy_linaro_android_image(BaseAction): def run(self, boot, system, data, use_cache=True): @@ -37,10 +38,16 @@ client.boot_master_image() print "Waiting for network to come up" - client.wait_network_up() + try: + client.wait_network_up() + except: + raise CriticalError("Network can't probe up when deployment") - boot_tbz2, system_tbz2, data_tbz2 = self.download_tarballs(boot, - system, data, use_cache) + try: + boot_tbz2, system_tbz2, data_tbz2 = self.download_tarballs(boot, + system, data, use_cache) + except: + raise CriticalError("Package can't download when deployment") boot_tarball = boot_tbz2.replace(LAVA_IMAGE_TMPDIR, '') system_tarball = system_tbz2.replace(LAVA_IMAGE_TMPDIR, '') @@ -58,7 +65,7 @@ self.deploy_linaro_android_testrootfs(system_url) self.purge_linaro_android_sdcard() except: - raise + raise CriticalError("Android deployment failed") finally: shutil.rmtree(self.tarball_dir) === modified file 'lava_dispatcher/actions/deploy.py' --- lava_dispatcher/actions/deploy.py 2011-07-06 17:16:22 +0000 +++ lava_dispatcher/actions/deploy.py 2011-07-20 09:41:30 +0000 @@ -27,6 +27,7 @@ from lava_dispatcher.actions import BaseAction from lava_dispatcher.config import LAVA_IMAGE_TMPDIR, LAVA_IMAGE_URL, MASTER_STR from lava_dispatcher.utils import download, download_with_cache +from lava_dispatcher.client import CriticalError class cmd_deploy_linaro_image(BaseAction): @@ -39,8 +40,16 @@ client.boot_master_image() print "Waiting for network to come up" - client.wait_network_up() - boot_tgz, root_tgz = self.generate_tarballs(hwpack, rootfs, use_cache) + try: + client.wait_network_up() + except: + raise CriticalError("Network can't probe up when deployment") + + try: + boot_tgz, root_tgz = self.generate_tarballs(hwpack, rootfs, + use_cache) + except: + raise CriticalError("Deployment tarballs preparation failed") boot_tarball = boot_tgz.replace(LAVA_IMAGE_TMPDIR, '') root_tarball = root_tgz.replace(LAVA_IMAGE_TMPDIR, '') boot_url = '/'.join(u.strip('/') for u in [ @@ -51,7 +60,7 @@ self.deploy_linaro_rootfs(root_url) self.deploy_linaro_bootfs(boot_url) except: - raise + raise CriticalError("Deployment failed") finally: shutil.rmtree(self.tarball_dir) === modified file 'lava_dispatcher/actions/lava-test.py' --- lava_dispatcher/actions/lava-test.py 2011-07-06 18:47:51 +0000 +++ lava_dispatcher/actions/lava-test.py 2011-07-20 05:27:33 +0000 @@ -92,7 +92,7 @@ try: client.run_shell_command( 'chroot /mnt/root lava-test help', - response="list-tests") + response="list-tests", timeout=10) except: raise OperationFailed("lava-test deployment failed") === modified file 'lava_dispatcher/android_client.py' --- lava_dispatcher/android_client.py 2011-06-29 10:22:40 +0000 +++ lava_dispatcher/android_client.py 2011-07-20 04:45:23 +0000 @@ -19,7 +19,7 @@ import pexpect import sys -from lava_dispatcher.client import LavaClient +from lava_dispatcher.client import LavaClient, OperationFailed from lava_dispatcher.android_config import BOARDS, TESTER_STR class LavaAndroidClient(LavaClient): === modified file 'lava_dispatcher/client.py' --- lava_dispatcher/client.py 2011-07-01 06:08:43 +0000 +++ lava_dispatcher/client.py 2011-07-20 05:27:33 +0000 @@ -153,15 +153,27 @@ def getvalue(self): return self.serialio.getvalue() - -class NetworkError(Exception): +class DispatcherError(Exception): + """ + Base exception and error class for dispatcher + """ + +class CriticalError(DispatcherError): + """ + The critical error + """ + +class GeneralError(DispatcherError): + """ + The non-critical error + """ + +class NetworkError(CriticalError): """ This is used when a network error occurs, such as failing to bring up the network interface on the client """ - -class OperationFailed(Exception): +class OperationFailed(GeneralError): pass -