[Branch,~linaro-validation/lava-dispatcher/trunk] Rev 84: Improved transferring of results from Spring Zhang

Message ID 20110816001712.14026.39221.launchpad@ackee.canonical.com
State Accepted
Headers show

Commit Message

Paul Larson Aug. 16, 2011, 12:17 a.m.
Merge authors:
  Spring Zhang (qzhang)
Related merge proposals:
  https://code.launchpad.net/~qzhang/lava-dispatcher/reliable-result-transfer/+merge/71341
  proposed by: Spring Zhang (qzhang)
  review: Approve - Paul Larson (pwlars)
  review: Resubmit - Spring Zhang (qzhang)
------------------------------------------------------------
revno: 84 [merge]
committer: Paul Larson <paul.larson@linaro.org>
branch nick: lava-dispatcher
timestamp: Tue 2011-08-16 01:15:08 +0100
message:
  Improved transferring of results from Spring Zhang
modified:
  lava_dispatcher/actions/__init__.py
  lava_dispatcher/actions/launch_control.py
  lava_dispatcher/android_client.py
  lava_dispatcher/android_config.py
  lava_dispatcher/client.py
  lava_dispatcher/config.py
  lava_dispatcher/utils.py


--
lp:lava-dispatcher
https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk

You are subscribed to branch lp:lava-dispatcher.
To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk/+edit-subscription

Patch

=== modified file 'lava_dispatcher/actions/__init__.py'
--- lava_dispatcher/actions/__init__.py	2011-06-23 19:38:42 +0000
+++ lava_dispatcher/actions/__init__.py	2011-08-16 00:15:08 +0000
@@ -34,8 +34,6 @@ 
 
 
 class BaseAndroidAction(BaseAction):
-    network_interface = "eth0"
-
     def __init__(self, context):
         self.context = context
 

=== modified file 'lava_dispatcher/actions/launch_control.py'
--- lava_dispatcher/actions/launch_control.py	2011-06-27 04:55:08 +0000
+++ lava_dispatcher/actions/launch_control.py	2011-08-16 00:15:08 +0000
@@ -20,11 +20,15 @@ 
 # along
 # with this program; if not, see <http://www.gnu.org/licenses>.
 
+import os
 import json
+import shutil
+import tarfile
 from lava_dispatcher.actions import BaseAction
 from lava_dispatcher.config import LAVA_RESULT_DIR, MASTER_STR, LAVA_SERVER_IP
-import socket
-from threading import Thread
+from lava_dispatcher.config import LAVA_IMAGE_TMPDIR
+from lava_dispatcher.client import NetworkError
+from lava_dispatcher.utils import download
 import time
 import xmlrpclib
 from subprocess import call
@@ -35,34 +39,21 @@ 
         srv = xmlrpclib.ServerProxy(xmlrpc_url,
                 allow_none=True, use_datetime=True)
 
-        client = self.client
-        call("cd /tmp/%s/; ls *.bundle > bundle.lst" % LAVA_RESULT_DIR,
-            shell=True)
-
-        t = ResultUploader()
-        t.start()
-        call('cd /tmp/%s/; cat bundle.lst |nc %s %d' % (LAVA_RESULT_DIR,
-            LAVA_SERVER_IP, t.get_port()), shell=True)
-        t.join()
-
-        bundle_list = t.get_data().strip().splitlines()
-        #Upload bundle files to server
-        for bundle in bundle_list:
-            t = ResultUploader()
-            t.start()
-            call('cat /tmp/%s/%s | nc %s %s' % (LAVA_RESULT_DIR, bundle,
-                LAVA_SERVER_IP, t.get_port()), shell = True)
-            t.join()
-            content = t.get_data()
+        #Upload bundle files to dashboard
+        bundle_list = os.listdir("/tmp/%s" % LAVA_RESULT_DIR)
+        for bundle_name in bundle_list:
+            bundle = "/tmp/%s/%s" % (LAVA_RESULT_DIR, bundle_name)
+            f = open(bundle) 
+            content = f.read()
+            f.close()
             try:
                 srv.put(content, bundle, stream)
             except xmlrpclib.Fault, err:
                 print "xmlrpclib.Fault occurred"
                 print "Fault code: %d" % err.faultCode
                 print "Fault string: %s" % err.faultString
-
             # After uploading, remove the bundle file at the host side
-            call('rm /tmp/%s/%s' % (LAVA_RESULT_DIR, bundle), shell=True)
+            os.remove(bundle)
 
 
 class cmd_submit_results(BaseAction):
@@ -96,43 +87,41 @@ 
                 LAVA_RESULT_DIR), response = MASTER_STR)
         client.run_shell_command('umount /mnt/root', response = MASTER_STR)
 
-        #Upload bundle list-bundle.lst
-        client.run_shell_command('cd /tmp/%s' % LAVA_RESULT_DIR,
-            response = MASTER_STR)
-        client.run_shell_command('ls *.bundle > bundle.lst',
-            response = MASTER_STR)
-
-        t = ResultUploader()
-        t.start()
-        #XXX: Odd problem where we sometimes get stuck here.  This is just
-        #     a hacky workaround to see if it's a race
-        time.sleep(60)
-        client.run_shell_command(
-            'cat bundle.lst |nc %s %d' % (LAVA_SERVER_IP, t.get_port()),
-            response = MASTER_STR)
-        t.join()
-
-        bundle_list = t.get_data().strip().splitlines()
-
+        #Create tarball of all results
+        client.run_shell_command('cd /tmp', response=MASTER_STR)
+        client.run_shell_command('tar czf /tmp/lava_results.tgz -C /tmp/%s .'
+                % LAVA_RESULT_DIR, response=MASTER_STR)
+
+        master_ip = client.get_master_ip()
+        if master_ip == None:
+            raise NetworkError("Getting master image IP address failed")
+        # Set 80 as server port
+        client.run_shell_command('python -m SimpleHTTPServer 80 &> /dev/null &',
+                response=MASTER_STR)
+        time.sleep(3)
+
+        result_tarball = "http://%s/lava_results.tgz" % master_ip
+        tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR)
+        os.chmod(tarball_dir, 0755)
+        #FIXME: need to consider exception?
+        result_path = download(result_tarball, tarball_dir)
+        id = client.proc.expect([MASTER_STR, pexpect.TIMEOUT, pexpect.EOF], 
+                timeout=3)
+        client.run_shell_command('kill %1', response=MASTER_STR)
+
+        tar = tarfile.open(result_path)
+        for tarinfo in tar:
+            if os.path.splitext(tarinfo.name)[1] == ".bundle":
+                f = tar.extractfile(tarinfo)
+                content = f.read()
+                f.close()
+                self.all_bundles.append(json.loads(content))
+        tar.close()
+        shutil.rmtree(tarball_dir)
+        
         #flush the serial log
         client.run_shell_command("")
 
-        #Upload bundle files to server
-        for bundle in bundle_list:
-            t = ResultUploader()
-            t.start()
-            #XXX: Odd problem where we sometimes get stuck here.  This is just
-            #     a hacky workaround to see if it's a race
-            time.sleep(60)
-            client.run_shell_command(
-                'cat /tmp/%s/%s | nc %s %s' % (LAVA_RESULT_DIR, bundle,
-                    LAVA_SERVER_IP, t.get_port()),
-                response = MASTER_STR)
-            t.join()
-            content = t.get_data()
-
-            self.all_bundles.append(json.loads(content))
-
         main_bundle = self.combine_bundles()
         self.context.test_data.add_seriallog(
             self.context.client.get_seriallog())
@@ -156,32 +145,3 @@ 
             test_runs += bundle['test_runs']
         return main_bundle
 
-class ResultUploader(Thread):
-    """
-    Simple HTTP Server for uploading bundles
-    """
-    def __init__(self):
-        """
-        if no filename specified, just get uploaded data
-        """
-        Thread.__init__(self)
-        self.data = ""
-        self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        self.s.bind(('', 0))
-
-    def get_port(self):
-        return self.s.getsockname()[1]
-
-    def get_data(self):
-        return self.data
-
-    def run(self):
-        self.s.listen(1)
-        conn, addr = self.s.accept()
-        while(1):
-            #10KB per time
-            data = conn.recv(10240)
-            if not data:
-                break
-            self.data = self.data + data
-        self.s.close()

=== modified file 'lava_dispatcher/android_client.py'
--- lava_dispatcher/android_client.py	2011-07-20 04:45:23 +0000
+++ lava_dispatcher/android_client.py	2011-08-16 00:15:08 +0000
@@ -106,17 +106,18 @@ 
 
     def check_adb_status(self):
         # XXX: IP could be assigned in other way in the validation farm
-        network_interface = self.board.network_interface 
+        network_interface = self.board.default_network_interface 
         try:
             self.run_shell_command('netcfg %s dhcp' % \
-                network_interface, response = TESTER_STR, timeout = 60)
+                default_network_interface, response = TESTER_STR,
+                timeout = 60)
         except:
-            print "netcfg %s dhcp exception" % network_interface
+            print "netcfg %s dhcp exception" % default_network_interface
             return False
 
         # Check network ip and setup adb connection
         ip_pattern = "(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})"
-        cmd = "ifconfig %s" % network_interface
+        cmd = "ifconfig %s" % default_network_interface
         self.proc.sendline('')
         self.proc.sendline(cmd)
         try:

=== modified file 'lava_dispatcher/android_config.py'
--- lava_dispatcher/android_config.py	2011-07-21 16:47:16 +0000
+++ lava_dispatcher/android_config.py	2011-08-16 00:15:08 +0000
@@ -31,7 +31,6 @@ 
         "init=/init androidboot.console=ttyO2'",
         "boot"]
     type = "beagle"
-    network_interface = "eth0"
 
 
 class PandaBoard(Board):
@@ -47,7 +46,6 @@ 
         "init=/init androidboot.console=ttyO2'",
         "boot"]
     type = "panda"
-    network_interface = "eth0"
 
 
 #Here, it still needs to maintain a map from boardid to board, for there is only

=== modified file 'lava_dispatcher/client.py'
--- lava_dispatcher/client.py	2011-07-21 16:47:16 +0000
+++ lava_dispatcher/client.py	2011-08-16 00:15:08 +0000
@@ -126,6 +126,25 @@ 
                 return
         raise NetworkError
 
+    def get_master_ip(self):
+        #get master image ip address
+        #tty device uses minimal match, see pexpect wiki
+        #pattern1 = ".*\n(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})"
+        pattern1 = "(\d?\d?\d?\.\d?\d?\d?\.\d?\d?\d?\.\d?\d?\d?)"
+        cmd = ("ifconfig %s | grep 'inet addr' | awk -F: '{print $2}' |"
+                "awk '{print $1}'" % self.board.default_network_interface)
+        self.proc.sendline(cmd)
+        #if running from ipython, it needs another Enter, don't know why
+        #self.proc.sendline("")
+        id = self.proc.expect([pattern1, pexpect.EOF, 
+            pexpect.TIMEOUT], timeout=5)
+        print "\nid=%s" %id
+        if id == 0:
+            ip = self.proc.match.groups()[0]
+            return ip
+        else:
+            return None
+
     def export_display(self):
         #export the display, ignore errors on non-graphical images
         self.run_shell_command("su - linaro -c 'DISPLAY=:0 xhost local:'",

=== modified file 'lava_dispatcher/config.py'
--- lava_dispatcher/config.py	2011-07-21 16:47:16 +0000
+++ lava_dispatcher/config.py	2011-08-16 00:15:08 +0000
@@ -32,6 +32,7 @@ 
     boot_part = 1
     # root partition number, counting from 1
     root_part = 2
+    default_network_interface = "eth0"
 
 class BeagleBoard(Board):
     uboot_cmds = ["mmc init",

=== modified file 'lava_dispatcher/utils.py'
--- lava_dispatcher/utils.py	2011-06-27 04:55:08 +0000
+++ lava_dispatcher/utils.py	2011-08-15 10:44:10 +0000
@@ -32,7 +32,7 @@ 
         filename = os.path.join(path,filename)
     fd = open(filename, "w")
     try:
-        response = urllib2.urlopen(urllib2.quote(url, safe=":/"))
+        response = urllib2.urlopen(urllib2.quote(url, safe=":/"), timeout=5)
         fd = open(filename, 'wb')
         shutil.copyfileobj(response,fd,0x10000)
         fd.close()