=== modified file 'fetch_image_ui.py'
--- fetch_image_ui.py	2011-07-21 20:41:51 +0000
+++ fetch_image_ui.py	2011-07-22 16:42:58 +0000
@@ -1134,8 +1134,15 @@
                              wx.ALIGN_LEFT | wx.LEFT | wx.RIGHT | wx.TOP,
                              5)
 
+        self.box2 = wx.BoxSizer(wx.VERTICAL)
+        self.blank = wx.StaticText(self, label="")  # Just like a spacer GIF...
+        self.messages = wx.StaticText(self, label="")
+        self.box2.Add(self.blank, 0, wx.ALIGN_LEFT | wx.ALL, 5)
+        self.box2.Add(self.messages, 0, wx.ALIGN_LEFT | wx.ALL, 5)
+
         self.sizer.Add(self.box1, 0, wx.ALIGN_LEFT | wx.ALL, 5)
         self.sizer.Add(self.status_grid, 0, wx.ALIGN_LEFT | wx.ALL, 5)
+        self.sizer.Add(self.box2, 0, wx.ALIGN_LEFT | wx.ALL, 5)
         self.SetSizerAndFit(self.sizer)
         self.sizer.Fit(self)
         self.Move((50, 50))
@@ -1233,7 +1240,7 @@
             elif event[0] == "end":
                 self.event_end(event[1])
 
-            elif event == "terminate":
+            elif event == "terminate" or event[0] == "abort":
                 # Process complete. Enable next button.
                 self.wizard.FindWindowById(wx.ID_FORWARD).Enable()
                 self.populate_file_system_status.SetLabel("Done")
@@ -1242,6 +1249,9 @@
             elif event[0] == "update":
                 self.event_update(event[1], event[2], event[3])
 
+            elif event[0] == "message":
+                self.messages.SetLabel(event[1])
+
             else:
                 print >> sys.stderr, "timer_ping: Unhandled event", event
 
@@ -1265,7 +1275,7 @@
     #--- Event(s) ---
     def event_start(self, event):
         if event == "download":
-            self.downloading_files_status.SetLabel("Downloading")
+            pass
         elif event == "unpack":
             self.unpacking_files_status.SetLabel("Running")
         elif event == "installing packages":

=== modified file 'linaro_image_tools/FetchImage.py'
--- linaro_image_tools/FetchImage.py	2011-07-21 20:41:51 +0000
+++ linaro_image_tools/FetchImage.py	2011-07-22 18:10:03 +0000
@@ -39,293 +39,22 @@
 import utils
 
 
-class FileHandler():
-    """Downloads files and creates images from them by calling
-    linaro-media-create"""
-    def __init__(self):
-        import xdg.BaseDirectory as xdgBaseDir
-        self.homedir = os.path.join(xdgBaseDir.xdg_config_home,
-                                     "linaro",
-                                     "image-tools",
-                                     "fetch_image")
-
-        self.cachedir = os.path.join(xdgBaseDir.xdg_cache_home,
-                                     "linaro",
-                                     "image-tools",
-                                     "fetch_image")
-
-    def has_key_and_evaluates_True(self, dictionary, key):
-        return bool(key in dictionary and dictionary[key])
-
-    def append_setting_to(self, list, dictionary, key, setting_name=None):
-        if not setting_name:
-            setting_name = "--" + key
-
-        if self.has_key_and_evaluates_True(dictionary, key):
-            list.append(setting_name)
-            list.append(dictionary[key])
-
-    def build_lmc_command(self,
-                          binary_file,
-                          hwpack_file,
-                          settings,
-                          tools_dir,
-                          hwpack_verified,
-                          run_in_gui=False):
-
-        import linaro_image_tools.utils
-
-        args = ["pkexec"]
-
-        # Prefer a local linaro-media-create (from a bzr checkout for instance)
-        # to an installed one
-        lmc_command = linaro_image_tools.utils.find_command(
-                                                    'linaro-media-create',
-                                                    tools_dir)
-
-        if lmc_command:
-            args.append(os.path.abspath(lmc_command))
-        else:
-            args.append("linaro-media-create")
-
-        if run_in_gui:
-            args.append("--nocheck-mmc")
-
-        if hwpack_verified:
-            # We verify the hwpack before calling linaro-media-create because
-            # these tools run linaro-media-create as root and to get the GPG
-            # signature verification to work, root would need to have GPG set
-            # up with the Canonical Linaro Image Build Automatic Signing Key
-            # imported. It seems far more likely that users will import keys
-            # as themselves, not root!
-            args.append("--hwpack-force-yes")
-
-        if 'rootfs' in settings and settings['rootfs']:
-            args.append("--rootfs")
-            args.append(settings['rootfs'])
-
-        assert(self.has_key_and_evaluates_True(settings, 'image_file') ^
-               self.has_key_and_evaluates_True(settings, 'mmc')), ("Please "
-               "specify either an image file, or an mmc target, not both.")
-
-        self.append_setting_to(args, settings, 'mmc')
-        self.append_setting_to(args, settings, 'image_file')
-        self.append_setting_to(args, settings, 'swap_size')
-        self.append_setting_to(args, settings, 'swap_file')
-        self.append_setting_to(args, settings, 'yes_to_mmc_selection',
-                                               '--nocheck_mmc')
-
-        args.append("--dev")
-        args.append(settings['hardware'])
-        args.append("--binary")
-        args.append(binary_file)
-        args.append("--hwpack")
-        args.append(hwpack_file)
-
-        logging.info(" ".join(args))
-
-        return args
-
-    def get_sig_files(self, downloaded_files):
-        """
-        Find sha1sum.txt files in downloaded_files, append ".asc" and return
-        list.
-
-        The reason for not just searching for .asc files is because if they
-        don't exist, utils.verify_file_integrity(sig_files) won't check the
-        sha1sums that do exist, and they are useful to us. Trying to check
-        a GPG signature that doesn't exist just results in the signature check
-        failing, which is correct and we act accordingly.
-        """
-
-        sig_files = []
-
-        for filename in downloaded_files.values():
-            if re.search(r"sha1sums\.txt$", filename):
-                sig_files.append(filename + ".asc")
-
-        return sig_files
-
-    def create_media(self, image_url, hwpack_url, settings, tools_dir):
-        """Create a command line for linaro-media-create based on the settings
-        provided then run linaro-media-create, either in a separate thread
-        (GUI mode) or in the current one (CLI mode)."""
-
-        sha1sums_urls = self.sha1sum_file_download_list(image_url,
-                                                        hwpack_url,
-                                                        settings)
-
-        sha1sum_files = self.download_files(sha1sums_urls,
-                                            settings)
-
-        to_download = self.generate_download_list(image_url,
-                                                  hwpack_url,
-                                                  sha1sum_files)
-
-        downloaded_files = self.download_files(to_download,
-                                               settings)
-
-        sig_files = self.get_sig_files(downloaded_files)
-
-        verified_files, gpg_sig_ok = utils.verify_file_integrity(sig_files)
-
-        hwpack = os.path.basename(downloaded_files[hwpack_url])
-        hwpack_verified = (hwpack in verified_files) and gpg_sig_ok
-
-        lmc_command = self.build_lmc_command(downloaded_files[image_url],
-                                             downloaded_files[hwpack_url],
-                                             settings,
-                                             tools_dir,
-                                             hwpack_verified)
-
-        self.create_process = subprocess.Popen(lmc_command)
-        self.create_process.wait()
-
-    class LinaroMediaCreate(threading.Thread):
-        """Thread class for running linaro-media-create"""
-        def __init__(self,
-                     image_url,
-                     hwpack_url,
-                     file_handler,
-                     event_queue,
-                     settings,
-                     tools_dir):
-
-            threading.Thread.__init__(self)
-
-            self.image_url      = image_url
-            self.hwpack_url     = hwpack_url
-            self.file_handler   = file_handler
-            self.event_queue    = event_queue
-            self.settings       = settings
-            self.tools_dir      = tools_dir
-
-        def run(self):
-            """
-            1. Download required files.
-            2. Build linaro-media-create command
-            3. Start linaro-media-create and look for lines in the output that:
-               1. Tell us that an event has happened that we can use to update
-                  the UI progress.
-               2. Tell us that linaro-media-create is asking a question that
-                  needs to be re-directed to the GUI
-            """
-
-            sha1sums_urls = self.file_handler.sha1sum_file_download_list(
-                                                               self.image_url,
-                                                               self.hwpack_url,
-                                                               self.settings)
-
-            self.event_queue.put(("update",
-                                  "download",
-                                  "message",
-                                  "Calculating download size"))
-
-            sha1sum_files = self.file_handler.download_files(sha1sums_urls,
-                                                             self.settings)
-
-            to_download = self.file_handler.generate_download_list(
-                                                            self.image_url,
-                                                            self.hwpack_url,
-                                                            sha1sum_files)
-
-            downloaded_files = self.file_handler.download_files(
-                                                            to_download,
-                                                            self.settings,
-                                                            self.event_queue)
-
-            sig_files = self.file_handler.get_sig_files(downloaded_files)
-
-            verified_files, gpg_sig_ok = utils.verify_file_integrity(sig_files)
-
-            hwpack = os.path.basename(downloaded_files[self.hwpack_url])
-            hwpack_verified = (hwpack in verified_files) and gpg_sig_ok
-
-            lmc_command = self.file_handler.build_lmc_command(
-                                            downloaded_files[self.image_url],
-                                            downloaded_files[self.hwpack_url],
-                                            self.settings,
-                                            self.tools_dir,
-                                            hwpack_verified,
-                                            True)
-
-            self.create_process = subprocess.Popen(lmc_command,
-                                                   stdin=subprocess.PIPE,
-                                                   stdout=subprocess.PIPE,
-                                                   stderr=subprocess.STDOUT)
-
-            self.line                       = ""
-            self.saved_lines                = ""
-            self.save_lines                 = False
-            self.state                      = None
-            self.waiting_for_event_response = False
-            self.event_queue.put(("start", "unpack"))
-
-            while(1):
-                if self.create_process.poll() != None:   # linaro-media-create
-                                                         # has finished.
-                    self.event_queue.put(("terminate"))  # Tell the GUI
-                    return                               # Terminate the thread
-
-                self.input = self.create_process.stdout.read(1)
-
-                # We build up lines by extracting data from linaro-media-create
-                # a single character at a time. This is because if we fetch
-                # whole lines, using Popen.communicate a character is
-                # automatically sent back to the running process, which if the
-                # process is waiting for user input, can trigger a default
-                # action. By using stdout.read(1) we pick off a character at a
-                # time and avoid this problem.
-                if self.input == '\n':
-                    if self.save_lines:
-                        self.saved_lines += self.line
-
-                    self.line = ""
-                else:
-                    self.line += self.input
-
-                if self.line == "Updating apt package lists ...":
-                    self.event_queue.put(("end", "unpack"))
-                    self.event_queue.put(("start", "installing packages"))
-                    self.state = "apt"
-
-                elif(self.line ==
-                     "WARNING: The following packages cannot be authenticated!"):
-                    self.saved_lines = ""
-                    self.save_lines = True
-
-                elif(self.line ==
-                     "Install these packages without verification [y/N]? "):
-                    self.saved_lines = re.sub("WARNING: The following packages"
-                                              " cannot be authenticated!",
-                                              "", self.saved_lines)
-                    self.event_queue.put(("start",
-                                          "unverified_packages:"
-                                          + self.saved_lines))
-                    self.line = ""
-                    self.waiting_for_event_response = True  # Wait for restart
-
-                elif self.line == "Do you want to continue [Y/n]? ":
-                    self.send_to_create_process("y")
-                    self.line = ""
-
-                elif self.line == "Done" and self.state == "apt":
-                    self.state = "create file system"
-                    self.event_queue.put(("end", "installing packages"))
-                    self.event_queue.put(("start", "create file system"))
-
-                elif(    self.line == "Created:"
-                     and self.state == "create file system"):
-                    self.event_queue.put(("end", "create file system"))
-                    self.event_queue.put(("start", "populate file system"))
-                    self.state = "populate file system"
-
-                while self.waiting_for_event_response:
-                    time.sleep(0.2)
-
-        def send_to_create_process(self, text):
-            print >> self.create_process.stdin, text
-            self.waiting_for_event_response = False
+class DownloadManager():
+    def __init__(self, cachedir):
+        self.cachedir           = cachedir
+        self.image_url          = None
+        self.hwpack_url         = None
+        self.settings           = None
+        self.event_queue        = None
+        self.to_download        = None
+        self.sha1_files         = None
+        self.gpg_files          = None
+        self.downloaded_files   = None
+        self.sig_files          = None
+        self.verified_files     = None
+        self.gpg_sig_ok         = None
+        self.have_sha1sums      = None
+        self.have_gpg_sigs      = None
 
     def name_and_path_from_url(self, url):
         """Return the file name and the path at which the file will be stored
@@ -350,7 +79,7 @@
                 response = urllib2.urlopen(url)
             except:
                 if trycount < maxtries - 1:
-                    print "Unable to connect to", url, "retrying in 5 seconds..."
+                    print "Unable to connect to", url, "retrying in 5 seconds."
                     time.sleep(5)
                     continue
                 else:
@@ -383,10 +112,7 @@
 
         return dir
 
-    def sha1sum_file_download_list(self,
-                                     image_url,
-                                     hwpack_url,
-                                     settings):
+    def sha1sum_file_download_list(self):
         """
         Need more than just the hwpack and OS image if we want signature
         verification to work, we need sha1sums, signatures for sha1sums
@@ -400,24 +126,21 @@
         downloads_list = []
 
         # Get list of files in OS image directory
-        image_dir  = self.list_files_in_dir_of_url(image_url)
-        hwpack_dir = self.list_files_in_dir_of_url(hwpack_url)
+        image_dir  = self.list_files_in_dir_of_url(self.image_url)
+        hwpack_dir = self.list_files_in_dir_of_url(self.hwpack_url)
 
         for link in image_dir:
             if re.search("sha1sums\.txt$", link):
                 downloads_list.append(link)
 
         for link in hwpack_dir:
-            if(    re.search(settings['hwpack'], link)
+            if(    re.search(self.settings['hwpack'], link)
                and re.search("sha1sums\.txt$", link)):
                 downloads_list.append(link)
 
         return downloads_list
 
-    def generate_download_list(self,
-                               image_url,
-                               hwpack_url,
-                               sha1sum_file_names):
+    def generate_download_list(self):
         """
         Generate a list of files based on what is in the sums files that
         we have downloaded. We may have downloaded some sig files based on
@@ -441,9 +164,9 @@
         linaro-media-create.
         """
 
-        downloads_list = [image_url, hwpack_url]
+        downloads_list = [self.image_url, self.hwpack_url]
 
-        for sha1sum_file_name in sha1sum_file_names.values():
+        for sha1sum_file_name in self.sha1_files.values():
             sha1sum_file = open(sha1sum_file_name)
 
             common_prefix = None
@@ -452,15 +175,15 @@
             for line in sha1sum_file:
                 line = line.split()    # line[1] is now the file name
 
-                if line[1] == os.path.basename(image_url):
+                if line[1] == os.path.basename(self.image_url):
                     # Found a line that matches an image or hwpack URL - keep
                     # the contents of this sig file
-                    common_prefix = os.path.dirname(image_url)
+                    common_prefix = os.path.dirname(self.image_url)
 
-                if line[1] == os.path.basename(hwpack_url):
+                if line[1] == os.path.basename(self.hwpack_url):
                     # Found a line that matches an image or hwpack URL - keep
                     # the contents of this sig file
-                    common_prefix = os.path.dirname(hwpack_url)
+                    common_prefix = os.path.dirname(self.hwpack_url)
 
             if common_prefix:
                 for file_name in files:
@@ -488,12 +211,15 @@
     def download_files(self,
                        downloads_list,
                        settings,
-                       event_queue=None):
+                       event_queue=None,
+                       force_download=False):
         """
         Download files specified in the downloads_list, which is a list of
         url, name tuples.
         """
 
+        force_download = settings["force_download"] or force_download
+
         downloaded_files = {}
 
         bytes_to_download = 0
@@ -502,7 +228,7 @@
             file_name, file_path = self.name_and_path_from_url(url)
 
             file_name = file_path + os.sep + file_name
-            if os.path.exists(file_name):
+            if os.path.exists(file_name) and not force_download:
                 continue  # If file already exists, don't download it
 
             response = self.urllib2_open(url)
@@ -523,7 +249,7 @@
             try:
                 path = self.download(url,
                                      event_queue,
-                                     settings["force_download"])
+                                     force_download)
             except Exception:
                 # Download error. Hardly matters what, we can't continue.
                 print "Unexpected error:", sys.exc_info()[0]
@@ -627,19 +353,412 @@
 
         return self.download(url, event_queue, force_download)
 
+    def get_sig_files(self):
+        """
+        Find sha1sum.txt files in downloaded_files, append ".asc" and return
+        list.
+
+        The reason for not just searching for .asc files is because if they
+        don't exist, utils.verify_file_integrity(sig_files) won't check the
+        sha1sums that do exist, and they are useful to us. Trying to check
+        a GPG signature that doesn't exist just results in the signature check
+        failing, which is correct and we act accordingly.
+        """
+
+        self.sig_files = []
+
+        for filename in self.downloaded_files.values():
+            if re.search(r"sha1sums\.txt$", filename):
+                self.sig_files.append(filename + ".asc")
+
+    def _download_sigs_gen_download_list(self, force_download=False):
+
+        to_download = self.sha1sum_file_download_list()
+        self.sha1_files = self.download_files(
+                            to_download, self.settings,
+                            force_download=force_download)
+
+        self.to_download = self.generate_download_list()
+
+        gpg_urls = [f for f in self.to_download if re.search('\.asc$', f)]
+        gpg_files = self.download_files(gpg_urls, self.settings,
+                                        force_download=force_download)
+
+    def _check_downloads(self):
+        self.get_sig_files()
+
+        self.verified_files, self.gpg_sig_ok = utils.verify_file_integrity(
+                                                            self.sig_files)
+
+        # Expect to have 2 sha1sum files (one for hwpack, one for OS bin)
+        self.have_sha1sums = len(self.sha1_files) ==2
+
+        # We expect 2 GPG signatures. Check that we have both of them.
+
+        self.have_gpg_sigs = (len(self.sig_files) == 2 and
+                              os.path.exists(self.sig_files[0]) and
+                              os.path.exists(self.sig_files[1]))
+
+    def _unverified_files(self):
+        """
+        Return a list of URLs of files that didn't get verified
+        """
+        verified_files = self.verified_files
+
+        unverified_files = []
+        for sig_file in self.sig_files:
+            name = os.path.basename(sig_file)
+            verified_files.append(name)
+            verified_files.append(name[0:-len('.asc')])
+        
+        # Generate a list of files that didn't verify
+        for url in self.to_download:
+            url_file = os.path.basename(url)
+            if url_file not in verified_files:
+                unverified_files.append(url)
+
+        return unverified_files
+
+    def download_files_and_verify(self, image_url, hwpack_url,
+                                  settings, event_queue=None):
+
+        self.image_url      = image_url
+        self.hwpack_url     = hwpack_url
+        self.settings       = settings
+        self.event_queue    = event_queue
+
+        self._download_sigs_gen_download_list()
+
+        self.downloaded_files = self.download_files(self.to_download,
+                                                    self.settings,
+                                                    self.event_queue)
+
+        self._check_downloads()
+
+        if(self.have_sha1sums and self.have_gpg_sigs
+           and not self.gpg_sig_ok):
+            # GPG signatures failed for sha1sum files. Re-download the
+            # sha1sums and GPG signatues. If the GPG signature then
+            # matches the sha1sums we will re-download any failing hwpack
+            # and OS binary files in the if below.
+
+            self._download_sigs_gen_download_list(force_download=True)
+            self._check_downloads()
+
+            if(self.have_sha1sums and self.have_gpg_sigs
+               and not self.gpg_sig_ok):
+                # If after re-trying the downloads we still can't get a GPG
+                # signature match on a sha1sum file (and both files exist)
+                # the abort.
+                message = "Package signature check failed. Aborting"
+                if self.event_queue:
+                    self.event_queue.put("message", message)
+                    self.event_queue.put("abort")
+                else:
+                    print >> sys.stderr, message
+
+                return [], False
+
+        if(self.have_sha1sums and 
+           self.gpg_sig_ok or not self.have_gpg_sigs):
+            # The GPG signature is OK, so the sha1sums file and GPG sig are
+            # both good. OR We have sha1sum files and no GPG sigs. We can
+            # Still validate downloads against the sha1sums, just not mark
+            # the packages as signed.
+
+            to_retry = self._unverified_files()
+
+            if len(to_retry):
+                if self.event_queue:
+                    self.event_queue.put(("message", "Retrying bad files"))
+                else:
+                    print "Re-downloading corrupt files"
+                # There are some files to re-download
+                redownloaded_files = self.download_files(
+                                           to_retry, self.settings,
+                                           self.event_queue,
+                                           force_download=True)
+
+                (self.verified_files,
+                 self.gpg_sig_ok) = utils.verify_file_integrity(self.sig_files)
+
+                to_retry = self._unverified_files()
+
+                if len(to_retry):
+                    # We can't get valid package downloads. Either the sha1sums
+                    # file was incorrectly generated or the download is
+                    # corrupt. Display a message to the user and quit.
+                    message = "Download retry failed. Aborting"
+                    if self.event_queue:
+                        self.event_queue.put("message", message)
+                        self.event_queue.put("abort")
+                    else:
+                        print >> sys.stderr, message
+
+                    return [], False
+
+        hwpack = os.path.basename(self.downloaded_files[hwpack_url])
+        hwpack_verified = (hwpack in self.verified_files) and self.gpg_sig_ok
+
+        if self.event_queue:  # Clear messages, if any, from GUI
+            self.event_queue.put(("message", ""))
+
+        return self.downloaded_files, hwpack_verified
+
+
+class FileHandler():
+    """Downloads files and creates images from them by calling
+    linaro-media-create"""
+    def __init__(self):
+        import xdg.BaseDirectory as xdgBaseDir
+        self.homedir = os.path.join(xdgBaseDir.xdg_config_home,
+                                     "linaro",
+                                     "image-tools",
+                                     "fetch_image")
+
+        self.cachedir = os.path.join(xdgBaseDir.xdg_cache_home,
+                                     "linaro",
+                                     "image-tools",
+                                     "fetch_image")
+
+        self.downloader = DownloadManager(self.cachedir)
+
+    def has_key_and_evaluates_True(self, dictionary, key):
+        return bool(key in dictionary and dictionary[key])
+
+    def append_setting_to(self, list, dictionary, key, setting_name=None):
+        if not setting_name:
+            setting_name = "--" + key
+
+        if self.has_key_and_evaluates_True(dictionary, key):
+            list.append(setting_name)
+            list.append(dictionary[key])
+
+    def build_lmc_command(self,
+                          binary_file,
+                          hwpack_file,
+                          settings,
+                          tools_dir,
+                          hwpack_verified,
+                          run_in_gui=False):
+
+        import linaro_image_tools.utils
+
+        args = ["pkexec"]
+
+        # Prefer a local linaro-media-create (from a bzr checkout for instance)
+        # to an installed one
+        lmc_command = linaro_image_tools.utils.find_command(
+                                                    'linaro-media-create',
+                                                    tools_dir)
+
+        if lmc_command:
+            args.append(os.path.abspath(lmc_command))
+        else:
+            args.append("linaro-media-create")
+
+        if run_in_gui:
+            args.append("--nocheck-mmc")
+
+        if hwpack_verified:
+            # We verify the hwpack before calling linaro-media-create because
+            # these tools run linaro-media-create as root and to get the GPG
+            # signature verification to work, root would need to have GPG set
+            # up with the Canonical Linaro Image Build Automatic Signing Key
+            # imported. It seems far more likely that users will import keys
+            # as themselves, not root!
+            args.append("--hwpack-force-yes")
+
+        if 'rootfs' in settings and settings['rootfs']:
+            args.append("--rootfs")
+            args.append(settings['rootfs'])
+
+        assert(self.has_key_and_evaluates_True(settings, 'image_file') ^
+               self.has_key_and_evaluates_True(settings, 'mmc')), ("Please "
+               "specify either an image file, or an mmc target, not both.")
+
+        self.append_setting_to(args, settings, 'mmc')
+        self.append_setting_to(args, settings, 'image_file')
+        self.append_setting_to(args, settings, 'swap_size')
+        self.append_setting_to(args, settings, 'swap_file')
+        self.append_setting_to(args, settings, 'yes_to_mmc_selection',
+                                               '--nocheck_mmc')
+
+        args.append("--dev")
+        args.append(settings['hardware'])
+        args.append("--binary")
+        args.append(binary_file)
+        args.append("--hwpack")
+        args.append(hwpack_file)
+
+        logging.info(" ".join(args))
+
+        return args
+
+    def create_media(self, image_url, hwpack_url, settings, tools_dir):
+        """Create a command line for linaro-media-create based on the settings
+        provided then run linaro-media-create, either in a separate thread
+        (GUI mode) or in the current one (CLI mode)."""
+
+        downloaded_files, hwpack_verified = (
+            self.downloader.download_files_and_verify(image_url, hwpack_url,
+                                                      settings))
+
+        if len(downloaded_files) == 0:
+            return
+
+        lmc_command = self.build_lmc_command(downloaded_files[image_url],
+                                             downloaded_files[hwpack_url],
+                                             settings,
+                                             tools_dir,
+                                             hwpack_verified)
+
+        self.create_process = subprocess.Popen(lmc_command)
+        self.create_process.wait()
+
+    class LinaroMediaCreate(threading.Thread):
+        """Thread class for running linaro-media-create"""
+        def __init__(self,
+                     image_url,
+                     hwpack_url,
+                     file_handler,
+                     event_queue,
+                     settings,
+                     tools_dir):
+
+            threading.Thread.__init__(self)
+
+            self.image_url      = image_url
+            self.hwpack_url     = hwpack_url
+            self.file_handler   = file_handler
+            self.downloader     = file_handler.downloader
+            self.event_queue    = event_queue
+            self.settings       = settings
+            self.tools_dir      = tools_dir
+
+        def run(self):
+            """
+            1. Download required files.
+            2. Build linaro-media-create command
+            3. Start linaro-media-create and look for lines in the output that:
+               1. Tell us that an event has happened that we can use to update
+                  the UI progress.
+               2. Tell us that linaro-media-create is asking a question that
+                  needs to be re-directed to the GUI
+            """
+
+            self.event_queue.put(("update",
+                                  "download",
+                                  "message",
+                                  "Downloading"))
+
+            files, hwpack_ok = self.downloader.download_files_and_verify(
+                                self.image_url, self.hwpack_url,
+                                self.settings, self.event_queue)
+
+            if len(files) == 0:
+                return
+
+            lmc_command = self.file_handler.build_lmc_command(
+                                            files[self.image_url],
+                                            files[self.hwpack_url],
+                                            self.settings,
+                                            self.tools_dir,
+                                            hwpack_ok,
+                                            True)
+
+            self.create_process = subprocess.Popen(lmc_command,
+                                                   stdin=subprocess.PIPE,
+                                                   stdout=subprocess.PIPE,
+                                                   stderr=subprocess.STDOUT)
+
+            self.line                       = ""
+            self.saved_lines                = ""
+            self.save_lines                 = False
+            self.state                      = None
+            self.waiting_for_event_response = False
+            self.event_queue.put(("start", "unpack"))
+
+            while(1):
+                if self.create_process.poll() != None:   # linaro-media-create
+                                                         # has finished.
+                    self.event_queue.put(("terminate"))  # Tell the GUI
+                    return                               # Terminate the thread
+
+                self.input = self.create_process.stdout.read(1)
+
+                # We build up lines by extracting data from linaro-media-create
+                # a single character at a time. This is because if we fetch
+                # whole lines, using Popen.communicate a character is
+                # automatically sent back to the running process, which if the
+                # process is waiting for user input, can trigger a default
+                # action. By using stdout.read(1) we pick off a character at a
+                # time and avoid this problem.
+                if self.input == '\n':
+                    if self.save_lines:
+                        self.saved_lines += self.line
+
+                    self.line = ""
+                else:
+                    self.line += self.input
+
+                if self.line == "Updating apt package lists ...":
+                    self.event_queue.put(("end", "unpack"))
+                    self.event_queue.put(("start", "installing packages"))
+                    self.state = "apt"
+
+                elif(self.line ==
+                     "WARNING: The following packages cannot be authenticated!"):
+                    self.saved_lines = ""
+                    self.save_lines = True
+
+                elif(self.line ==
+                     "Install these packages without verification [y/N]? "):
+                    self.saved_lines = re.sub("WARNING: The following packages"
+                                              " cannot be authenticated!",
+                                              "", self.saved_lines)
+                    self.event_queue.put(("start",
+                                          "unverified_packages:"
+                                          + self.saved_lines))
+                    self.line = ""
+                    self.waiting_for_event_response = True  # Wait for restart
+
+                elif self.line == "Do you want to continue [Y/n]? ":
+                    self.send_to_create_process("y")
+                    self.line = ""
+
+                elif self.line == "Done" and self.state == "apt":
+                    self.state = "create file system"
+                    self.event_queue.put(("end", "installing packages"))
+                    self.event_queue.put(("start", "create file system"))
+
+                elif(    self.line == "Created:"
+                     and self.state == "create file system"):
+                    self.event_queue.put(("end", "create file system"))
+                    self.event_queue.put(("start", "populate file system"))
+                    self.state = "populate file system"
+
+                while self.waiting_for_event_response:
+                    time.sleep(0.2)
+
+        def send_to_create_process(self, text):
+            print >> self.create_process.stdin, text
+            self.waiting_for_event_response = False
+
+
     def update_files_from_server(self, force_download=False,
                                  event_queue=None):
 
         settings_url     = "http://releases.linaro.org/fetch_image/fetch_image_settings.yaml"
         server_index_url = "http://releases.linaro.org/fetch_image/server_index.bz2"
 
-        self.settings_file = self.download_if_old(settings_url,
-                                                  event_queue,
-                                                  force_download)
+        self.settings_file = self.downloader.download_if_old(settings_url,
+                                                             event_queue,
+                                                             force_download)
 
-        self.index_file = self.download_if_old(server_index_url,
-                                               event_queue,
-                                               force_download)
+        self.index_file = self.downloader.download_if_old(server_index_url,
+                                                          event_queue,
+                                                          force_download)
 
         zip_search = re.search(r"^(.*)\.bz2$", self.index_file)
 
@@ -1153,9 +1272,13 @@
                            (date, hwpack))
 
             if len(builds):
-                # A hardware pack exists for that date, return what we found
-                # for binaries
-                return binaries
+                # A hardware pack exists for that date, return a list of builds
+                # that exist for both hwpack and binary
+                ret = []
+                for build in builds:
+                    if build in binaries:
+                        ret.append(build)
+                return ret
 
         # No hardware pack exists for the date requested, return empty table
         return []
@@ -1243,16 +1366,43 @@
             count = 0
             while(1):  # Just so we can try several times...
                 if(args['build'] == "latest"):
-                    # First result of SQL query is latest build (ORDER BY)
+                    # Start from today, add 1 day because
+                    # get_next_prev_day_with_builds aways searches from the
+                    # day passed to it, not including that day. This will give
+                    # us the latest build dated today or earlier with both
+                    # a hwpack and OS binary
+                    date = (datetime.date.today() +
+                            datetime.timedelta(days=1)).isoformat()
+
+                    _, past_date = self.get_next_prev_day_with_builds(
+                                           args['image'],
+                                           date,
+                                           [args['hwpack']])
+
+                    builds = self.get_binary_builds_on_day_from_db(
+                                    args['image'], past_date, [args['hwpack']])
+                    
+                    # Work out latest build
+                    build = None
+                    for b in builds:
+                        if build == None or b[0] > build:
+                            build = b[0]
+                        
+                    # We store dates in the DB in ISO format with the dashes
+                    # missing
+                    date = re.sub('-', '', past_date)
+
                     image_url = self.get_url("snapshot_binaries",
                                     [
-                                     ("image", args['image'])],
-                                    "ORDER BY date DESC, build DESC")
+                                     ("image", args['image']),
+                                     ("build", build),
+                                     ("date", date)])
 
                     hwpack_url = self.get_url("snapshot_hwpacks",
                                     [
-                                     ("hardware", args['hwpack'])],
-                                    "ORDER BY date DESC, build DESC")
+                                     ("hardware", args['hwpack']),
+                                     ("build", build),
+                                     ("date", date)])
 
                 else:
                     build_bits = args['build'].split(":")

