[Branch,~linaro-validation/lava-dispatcher/trunk] Rev 110: merge kernel inject branch

Message ID 20110915084413.13279.45889.launchpad@ackee.canonical.com
State Accepted
Headers show

Commit Message

Spring Zhang Sept. 15, 2011, 8:44 a.m.
Merge authors:
  Spring Zhang (qzhang)
Related merge proposals:
  https://code.launchpad.net/~qzhang/lava-dispatcher/another-inject-kernel/+merge/72927
  proposed by: Spring Zhang (qzhang)
  https://code.launchpad.net/~qzhang/lava-dispatcher/inject-kernel/+merge/69975
  proposed by: Spring Zhang (qzhang)
  review: Resubmit - Spring Zhang (qzhang)
  review: Needs Information - Paul Larson (pwlars)
------------------------------------------------------------
revno: 110 [merge]
committer: Spring Zhang <spring.zhang@linaro.org>
branch nick: another-inject-kernel
timestamp: Thu 2011-09-15 15:46:03 +0800
message:
  merge kernel inject branch
added:
  doc/android-new-kernel.json
  doc/lava-new-kernel.json
renamed:
  doc/lava-ltp-job => doc/lava-ltp-job.json
modified:
  lava_dispatcher/actions/android_deploy.py
  lava_dispatcher/actions/deploy.py
  lava_dispatcher/client.py
  lava_dispatcher/utils.py


--
lp:lava-dispatcher
https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk

You are subscribed to branch lp:lava-dispatcher.
To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk/+edit-subscription

Patch

=== added file 'doc/android-new-kernel.json'
--- doc/android-new-kernel.json	1970-01-01 00:00:00 +0000
+++ doc/android-new-kernel.json	2011-08-16 06:15:23 +0000
@@ -0,0 +1,37 @@ 
+{
+  "job_name": "android_new_kernel",
+  "image_type": "android",
+  "target": "panda01",
+  "timeout": 18000,
+  "actions": [
+    {
+      "command": "deploy_linaro_android_image",
+      "parameters":
+        {
+          "boot": "https://android-build.linaro.org/jenkins/job/linaro-android_leb-panda/61/artifact/build/out/target/product/pandaboard/boot.tar.bz2",
+          "system": "https://android-build.linaro.org/jenkins/job/linaro-android_leb-panda/61/artifact/build/out/target/product/pandaboard/system.tar.bz2",
+          "data": "https://android-build.linaro.org/jenkins/job/linaro-android_leb-panda/61/artifact/build/out/target/product/pandaboard/userdata.tar.bz2",
+          "pkg": "https://android-build.linaro.org/jenkins/job/linaro-android_leb-panda/171/artifact/build/out/target/product/pandaboard/boot.tar.bz2"
+        },
+      "metadata":
+        {
+          "rootfs.type": "android",
+          "rootfs.build": "61"
+        }
+    },
+    {
+      "command": "boot_linaro_android_image"
+    },
+    {
+      "command": "test_android_basic"
+    },
+    {
+      "command": "submit_results_on_host",
+      "parameters":
+        {
+          "server": "http://validation.linaro.org/launch-control",
+          "stream": "/anonymous/android-panda01-basic/"
+        }
+    }
+  ]
+}

=== renamed file 'doc/lava-ltp-job' => 'doc/lava-ltp-job.json'
=== added file 'doc/lava-new-kernel.json'
--- doc/lava-new-kernel.json	1970-01-01 00:00:00 +0000
+++ doc/lava-new-kernel.json	2011-08-18 10:40:22 +0000
@@ -0,0 +1,30 @@ 
+{
+  "job_name": "new-kernel",
+  "target": "mx53loco01",
+  "timeout": 6000,
+  "actions": [
+    {
+      "command": "deploy_linaro_image",
+      "parameters":
+        {
+          "rootfs":
+"http://snapshots.linaro.org/11.05-daily/linaro-nano/20110612/0/images/tar/nano-n-tar-20110612-0.tar.gz",
+          "hwpack":
+"http://snapshots.linaro.org/11.05-daily/linaro-hwpacks/lt-mx53loco/20110609/0/images/hwpack/hwpack_linaro-lt-mx53loco_20110609-0_armel_supported.tar.gz",
+          "kernel_matrix":[
+"http://pkgserver/original/linux-image-2.6.38-1000-linaro-lt-mx5_2.6.38-1000.7_armel.deb", "linux-image-2.6.38"]
+        }
+    },
+    {
+      "command": "boot_linaro_image"
+    },
+    {
+      "command": "submit_results",
+      "parameters":
+        {
+          "server": "http://validation.linaro.org/launch-control",
+          "stream": "/anonymous/testresult/"
+        }
+    }
+  ]
+}

=== modified file 'lava_dispatcher/actions/android_deploy.py'
--- lava_dispatcher/actions/android_deploy.py	2011-09-05 22:39:47 +0000
+++ lava_dispatcher/actions/android_deploy.py	2011-09-14 04:18:40 +0000
@@ -29,7 +29,7 @@ 
 from lava_dispatcher.client import CriticalError
 
 class cmd_deploy_linaro_android_image(BaseAction):
-    def run(self, boot, system, data, use_cache=True):
+    def run(self, boot, system, data, pkg=None, use_cache=True):
         LAVA_IMAGE_TMPDIR = self.context.lava_image_tmpdir
         LAVA_IMAGE_URL = self.context.lava_image_url
         client = self.client
@@ -49,8 +49,8 @@ 
             raise CriticalError("Unable to reach LAVA server, check network")
 
         try:
-            boot_tbz2, system_tbz2, data_tbz2 = self.download_tarballs(boot,
-                system, data, use_cache)
+            boot_tbz2, system_tbz2, data_tbz2, pkg_tbz2 = \
+                self.download_tarballs(boot, system, data, pkg, use_cache)
         except:
             tb = traceback.format_exc()
             client.sio.write(tb)
@@ -66,9 +66,16 @@ 
             LAVA_IMAGE_URL, system_tarball])
         data_url = '/'.join(u.strip('/') for u in [
             LAVA_IMAGE_URL, data_tarball])
+        if pkg_tbz2:
+            pkg_tarball = pkg_tbz2.replace(LAVA_IMAGE_TMPDIR, '')
+            pkg_url = '/'.join(u.strip('/') for u in [
+                LAVA_IMAGE_URL, pkg_tarball])
 
         try:
-            self.deploy_linaro_android_testboot(boot_url)
+            if pkg_tbz2:
+                self.deploy_linaro_android_testboot(boot_url, pkg_url)
+            else:
+                self.deploy_linaro_android_testboot(boot_url)
             self.deploy_linaro_android_testrootfs(system_url)
             self.purge_linaro_android_sdcard()
         except:
@@ -78,12 +85,14 @@ 
         finally:
             shutil.rmtree(self.tarball_dir)
 
-    def download_tarballs(self, boot_url, system_url, data_url, use_cache=True):
+    def download_tarballs(self, boot_url, system_url, data_url, pkg_url=None,
+            use_cache=True):
         """Download tarballs from a boot, system and data tarball url
 
         :param boot_url: url of the Linaro Android boot tarball to download
         :param system_url: url of the Linaro Android system tarball to download
         :param data_url: url of the Linaro Android data tarball to download
+        :param pkg_url: url of the custom kernel tarball to download
         :param use_cache: whether or not to use the cached copy (if it exists)
         """
         lava_cachedir = self.context.lava_cachedir
@@ -96,13 +105,21 @@ 
             boot_path = download_with_cache(boot_url, tarball_dir, lava_cachedir)
             system_path = download_with_cache(system_url, tarball_dir, lava_cachedir)
             data_path = download_with_cache(data_url, tarball_dir, lava_cachedir)
+            if pkg_url:
+                pkg_path = download_with_cache(pkg_url, tarball_dir)
+            else:
+                pkg_path = None
         else:
             boot_path = download(boot_url, tarball_dir)
             system_path = download(system_url, tarball_dir)
             data_path = download(data_url, tarball_dir)
-        return  boot_path, system_path, data_path
+            if pkg_url:
+                pkg_path = download(pkg_url, tarball_dir)
+            else:
+                pkg_path = None
+        return  boot_path, system_path, data_path, pkg_path
 
-    def deploy_linaro_android_testboot(self, boottbz2):
+    def deploy_linaro_android_testboot(self, boottbz2, pkgbz2=None):
         client = self.client
         client.run_cmd_master('mkfs.vfat /dev/disk/by-label/testboot '
                               '-n testboot')
@@ -111,6 +128,10 @@ 
         client.run_cmd_master('mount /dev/disk/by-label/testboot '
                               '/mnt/lava/boot')
         client.run_cmd_master('wget -qO- %s |tar --numeric-owner -C /mnt/lava -xjf -' % boottbz2)
+        if pkgbz2:
+            client.run_shell_command(
+                'wget -qO- %s |tar --numeric-owner -C /mnt/lava -xjf -' 
+                    % pkgbz2, response = MASTER_STR)
 
         self.recreate_uInitrd()
 

=== modified file 'lava_dispatcher/actions/deploy.py'
--- lava_dispatcher/actions/deploy.py	2011-09-08 23:35:05 +0000
+++ lava_dispatcher/actions/deploy.py	2011-09-15 07:46:03 +0000
@@ -15,11 +15,11 @@ 
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along
-# with this program; if not, see <http://www.gnu.org/licenses>.
+# along with this program; if not, see <http://www.gnu.org/licenses>.
 
 from commands import getoutput, getstatusoutput
 import os
+import sys
 import re
 import shutil
 import traceback
@@ -31,13 +31,15 @@ 
 
 
 class cmd_deploy_linaro_image(BaseAction):
-    def run(self, hwpack, rootfs, use_cache=True):
+    def run(self, hwpack, rootfs, kernel_matrix=None, use_cache=True):
         LAVA_IMAGE_TMPDIR = self.context.lava_image_tmpdir
         LAVA_IMAGE_URL = self.context.lava_image_url
         client = self.client
         print "deploying on %s" % client.hostname
         print "  hwpack: %s" % hwpack
         print "  rootfs: %s" % rootfs
+        if kernel_matrix:
+            print "  package: %s" % kernel_matrix[0]
         print "Booting master image"
         client.boot_master_image()
 
@@ -49,9 +51,17 @@ 
             client.sio.write(tb)
             raise CriticalError("Unable to reach LAVA server, check network")
 
+        if kernel_matrix:
+            hwpack = self.refresh_hwpack(kernel_matrix, hwpack, use_cache)
+            #make new hwpack downloadable
+            hwpack = hwpack.replace(LAVA_IMAGE_TMPDIR, '')
+            hwpack = '/'.join(u.strip('/') for u in [
+                LAVA_IMAGE_URL, hwpack])
+            print "  hwpack with new kernel: %s" % hwpack
+
         try:
-            boot_tgz, root_tgz = self.generate_tarballs(
-                hwpack, rootfs, use_cache)
+            boot_tgz, root_tgz = self.generate_tarballs(hwpack, rootfs, 
+                use_cache)
         except:
             tb = traceback.format_exc()
             client.sio.write(tb)
@@ -119,6 +129,7 @@ 
         self.tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR)
         tarball_dir = self.tarball_dir
         os.chmod(tarball_dir, 0755)
+        #fix me: if url is not http-prefix, copy it to tarball_dir
         if use_cache:
             hwpack_path = download_with_cache(hwpack_url, tarball_dir, lava_cachedir)
             rootfs_path = download_with_cache(rootfs_url, tarball_dir, lava_cachedir)
@@ -184,3 +195,45 @@ 
             'wget -qO- %s |tar --numeric-owner -C /mnt/boot -xzf -' % bootfs)
         client.run_cmd_master('umount /mnt/boot')
 
+    def refresh_hwpack(self, kernel_matrix, hwpack, use_cache=True):
+        client = self.client
+        LAVA_IMAGE_TMPDIR = self.context.lava_image_tmpdir
+        print "Deploying new kernel"
+        new_kernel = kernel_matrix[0]
+        deb_prefix = kernel_matrix[1]
+        filesuffix = new_kernel.split(".")[-1]
+
+        if filesuffix != "deb":
+            raise CriticalError("New kernel only support deb kernel package!")
+
+        # download package to local
+        tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR)
+        os.chmod(tarball_dir, 0755)
+        if use_cache:
+            kernel_path = download_with_cache(new_kernel, tarball_dir)
+            hwpack_path = download_with_cache(hwpack, tarball_dir)
+        else:
+            kernel_path = download(new_kernel, tarball_dir)
+            hwpack_path = download(hwpack, tarball_dir)
+
+        cmd = ("sudo linaro-hwpack-replace -t %s -p %s -r %s" 
+                % (hwpack_path, kernel_path, deb_prefix))
+
+        rc, output = getstatusoutput(cmd)
+        if rc:
+            shutil.rmtree(tarball_dir)
+            tb = traceback.format_exc()
+            client.sio.write(tb)
+            raise RuntimeError("linaro-hwpack-replace failed: %s" % output)
+
+        #fix it:l-h-r doesn't make a output option to specify the output hwpack,
+        #so it needs to do manually here
+
+        #remove old hwpack and leave only new hwpack in tarball_dir
+        os.remove(hwpack_path)
+        hwpack_list = os.listdir(tarball_dir)
+        for hp in hwpack_list:
+            if hp.split(".")[-1] == "gz":
+                new_hwpack_path = os.path.join(tarball_dir, hp)
+                return new_hwpack_path
+

=== modified file 'lava_dispatcher/client.py'
--- lava_dispatcher/client.py	2011-09-14 16:05:43 +0000
+++ lava_dispatcher/client.py	2011-09-15 07:02:08 +0000
@@ -130,7 +130,7 @@ 
         # Details: system PS1 is set in /etc/bash.bashrc and user PS1 is set in
         # /root/.bashrc, it is
         # "${debian_chroot:+($debian_chroot)}\u@\h:\w\$ "
-        self.proc.sendline('export PS1="$PS1 rc=$(echo \$?) "')
+        self.proc.sendline('export PS1="$PS1 [rc=$(echo \$?)]: "')
         self.proc.expect(self.tester_str)
 
     def enter_uboot(self):

=== modified file 'lava_dispatcher/utils.py'
--- lava_dispatcher/utils.py	2011-09-01 03:03:09 +0000
+++ lava_dispatcher/utils.py	2011-09-15 07:02:08 +0000
@@ -45,7 +45,11 @@ 
     if os.path.exists(cache_loc):
         filename = os.path.basename(cache_loc)
         file_location = os.path.join(path, filename)
-        os.link(cache_loc, file_location)
+        try:
+            os.link(cache_loc, file_location)
+        except OSError, err:
+            if err.errno == 18:
+                shutil.copy(cache_loc, file_location)
     else:
         file_location = download(url, path)
         try: