=== modified file 'linaro_image_tools/hwpack/builder.py'
--- linaro_image_tools/hwpack/builder.py	2012-08-28 17:34:16 +0000
+++ linaro_image_tools/hwpack/builder.py	2012-09-03 17:12:23 +0000
@@ -41,14 +41,20 @@
     PACKAGE_FIELD,
     SPL_FILE_FIELD,
     SPL_PACKAGE_FIELD,
+    COPY_FILES_FIELD,
 )
 
 # The fields that hold packages to be installed.
 PACKAGE_FIELDS = [PACKAGE_FIELD, SPL_PACKAGE_FIELD]
-# The fields that hold values that should be reset to newly calculated ones.
-# The values of the dictionary are the fields whose values should be reset.
-FIELDS_TO_CHANGE = {PACKAGE_FIELD: FILE_FIELD,
-                    SPL_PACKAGE_FIELD: SPL_FILE_FIELD}
+# Specification of files (boot related) to extract:
+# <field_containing_filepaths>: (<take_files_from_package>,
+#                                <put_into_this_hwpack_subdir>)
+# if <put_into_this_hwpack_subdir> is None, it will be <bootloader_name> for
+# global bootloader, or <board>-<bootloader_name> for board-specific
+# bootloader
+EXTRACT_FILES = {FILE_FIELD: (PACKAGE_FIELD, None),
+                 SPL_FILE_FIELD: (SPL_PACKAGE_FIELD, None),
+                 COPY_FILES_FIELD: (PACKAGE_FIELD, None)}
 
 
 logger = logging.getLogger(__name__)
@@ -74,19 +80,30 @@
         if self.tempdir is not None and os.path.exists(self.tempdir):
             shutil.rmtree(self.tempdir)
 
+    def get_path(self, package_file_name, file_name=''):
+        "Get package or file path in unpacker tmp dir."
+        package_dir = os.path.basename(package_file_name)
+        return os.path.join(self.tempdir, package_dir, file_name)
+
     def unpack_package(self, package_file_name):
         # We could extract only a single file, but since dpkg will pipe
         # the entire package through tar anyway we might as well extract all.
-        p = cmd_runner.run(["tar", "-C", self.tempdir, "-xf", "-"],
+        unpack_dir = self.get_path(package_file_name)
+        if not os.path.isdir(unpack_dir):
+            os.mkdir(unpack_dir)
+        p = cmd_runner.run(["tar", "-C", unpack_dir, "-xf", "-"],
                            stdin=subprocess.PIPE)
         cmd_runner.run(["dpkg", "--fsys-tarfile", package_file_name],
                        stdout=p.stdin).communicate()
         p.communicate()
 
     def get_file(self, package, file):
+        # File path passed here must not be absolute, or file from
+        # real filesystem will be referenced.
+        assert file and file[0] != '/'
         self.unpack_package(package)
         logger.debug("Unpacked package %s." % package)
-        temp_file = os.path.join(self.tempdir, file)
+        temp_file = self.get_path(package, file)
         assert os.path.exists(temp_file), "The file '%s' was " \
             "not found in the package '%s'." % (file, package)
         return temp_file
@@ -94,7 +111,7 @@
 
 class HardwarePackBuilder(object):
 
-    def __init__(self, config_path, version, local_debs):
+    def __init__(self, config_path, version, local_debs, out_name=None):
         try:
             with open(config_path) as fp:
                 self.config = Config(fp, allow_unset_bootloader=True)
@@ -110,6 +127,7 @@
         self.hwpack = None
         self.packages = None
         self.packages_added_to_hwpack = []
+        self.out_name = out_name
 
     def find_fetched_package(self, packages, wanted_package_name):
         wanted_package = None
@@ -123,7 +141,7 @@
         return wanted_package
 
     def add_file_to_hwpack(self, package, wanted_file, target_path):
-        if (package.name, target_path) in self.packages_added_to_hwpack:
+        if (package.name, wanted_file) in self.packages_added_to_hwpack:
             # Don't bother adding the same package more than once.
             return
 
@@ -148,34 +166,57 @@
         # Eliminate duplicates.
         return list(set(boot_packages))
 
-    def _set_new_values(self, config_dictionary):
-        """Loop through the bootloaders sections of a hwpack, also from the
-        boards section, changing the necessary values with the newly calculated
-        ones.
-
-        :param config_dictionary: The dictionary from the Config we need to
-                                    look into.
-        """
-        for key, value in config_dictionary.iteritems():
-            if isinstance(value, dict):
-                self._set_new_values(value)
-            else:
-                if key in FIELDS_TO_CHANGE.keys():
-                    if key == PACKAGE_FIELD:
-                        # Need to use the correct path for the packages.
-                        path = self.hwpack.U_BOOT_DIR
-                    else:
-                        path = self.hwpack.SPL_DIR
-                    change_field = FIELDS_TO_CHANGE.get(key)
-                    boot_package = value
-                    boot_file = config_dictionary.get(change_field)
-                    if boot_package is not None and boot_file is not None:
-                        package = self.find_fetched_package(
-                                    self.packages,
-                                    boot_package)
-                        file_to_add = self.add_file_to_hwpack(
-                                            package, boot_file, path)
-                        config_dictionary[change_field] = file_to_add
+    def extract_bootloader_files(self, board, bootloader_name,
+                                 bootloader_conf):
+        for key, value in bootloader_conf.iteritems():
+            if key in EXTRACT_FILES:
+                package_field, dest_path = EXTRACT_FILES[key]
+                if not dest_path:
+                    dest_path = bootloader_name
+                    if board:
+                        dest_path += "-" + board
+                # Dereference package field to get actual package name
+                package = bootloader_conf.get(package_field)
+                src_files = value
+
+                # Process scalar and list fields consistently below
+                field_value_scalar = False
+                if type(src_files) != type([]):
+                    src_files = [src_files]
+                    field_value_scalar = True
+
+                package_ref = self.find_fetched_package(
+                                self.packages, package)
+                added_files = []
+                for f in src_files:
+                    added_files.append(self.add_file_to_hwpack(
+                                        package_ref, f, dest_path))
+                # Store within-hwpack file paths with the same
+                # scalar/list type as original field.
+                if field_value_scalar:
+                    assert len(added_files) == 1
+                    added_files = added_files[0]
+                bootloader_conf[key] = added_files
+
+    def extract_files(self, config_dictionary, is_bootloader_config,
+                      board=None):
+        """Extract (boot) files based on EXTRACT_FILES spec and put
+        them into hwpack."""
+        self.remove_packages = []
+        if is_bootloader_config:
+            for bootl_name, bootl_conf in config_dictionary.iteritems():
+                self.extract_bootloader_files(board, bootl_name, bootl_conf)
+        else:
+            # This is board config
+            for board, board_conf in config_dictionary.iteritems():
+                bootloaders = board_conf['bootloaders']
+                self.extract_files(bootloaders, True, board)
+
+        # Clean up no longer needed packages.
+        for package in self.remove_packages:
+            if package in self.packages:
+                self.packages.remove(package)
+        self.remove_packages = []
 
     def build(self):
         for architecture in self.config.architectures:
@@ -186,7 +227,10 @@
             sources = self.config.sources
             with LocalArchiveMaker() as local_archive_maker:
                 self.hwpack.add_apt_sources(sources)
-                sources = sources.values()
+                if sources:
+                    sources = sources.values()
+                else:
+                    sources = []
                 self.packages = self.config.packages[:]
                 # Loop through multiple bootloaders.
                 # In V3 of hwpack configuration, all the bootloaders info and
@@ -226,10 +270,11 @@
                         # through both of them changing what is necessary.
                         if self.config.format.format_as_string == '3.0':
                             if self.config.bootloaders is not None:
-                                self._set_new_values(self.config.bootloaders)
+                                self.extract_files(self.config.bootloaders,
+                                                   True)
                                 metadata.bootloaders = self.config.bootloaders
                             if self.config.boards is not None:
-                                self._set_new_values(self.config.boards)
+                                self.extract_files(self.config.boards, False)
                                 metadata.boards = self.config.boards
                         else:
                             bootloader_package = None
@@ -275,9 +320,15 @@
                                     local_package.name)
                         self.hwpack.add_dependency_package(
                                 self.config.packages)
-                        with open(self.hwpack.filename(), 'w') as f:
+                        out_name = self.out_name
+                        if not out_name:
+                            out_name = self.hwpack.filename()
+                        with open(out_name, 'w') as f:
                             self.hwpack.to_file(f)
-                            logger.info("Wrote %s" % self.hwpack.filename())
-                        with open(self.hwpack.filename('.manifest.txt'),
-                                    'w') as f:
+                            logger.info("Wrote %s" % out_name)
+                        manifest_name = os.path.splitext(out_name)[0]
+                        if manifest_name.endswith('.tar'):
+                            manifest_name = os.path.splitext(manifest_name)[0]
+                        manifest_name += '.manifest.txt'
+                        with open(manifest_name, 'w') as f:
                             f.write(self.hwpack.manifest_text())

=== modified file 'linaro_image_tools/hwpack/hardwarepack.py'
--- linaro_image_tools/hwpack/hardwarepack.py	2012-07-26 14:52:12 +0000
+++ linaro_image_tools/hwpack/hardwarepack.py	2012-08-29 10:31:19 +0000
@@ -434,6 +434,7 @@
     SOURCES_LIST_GPG_DIRNAME = "sources.list.d.gpg"
     U_BOOT_DIR = "u-boot"
     SPL_DIR = "spl"
+    BOOT_DIR = "boot"
 
     def __init__(self, metadata):
         """Create a HardwarePack.
@@ -482,7 +483,8 @@
             to sources entries.
         :type sources: a dict mapping str to str
         """
-        self.sources.update(sources)
+        if sources:
+            self.sources.update(sources)
 
     def add_packages(self, packages):
         """Add packages to the hardware pack.

=== modified file 'linaro_image_tools/hwpack/packages.py'
--- linaro_image_tools/hwpack/packages.py	2012-06-13 14:26:02 +0000
+++ linaro_image_tools/hwpack/packages.py	2012-09-03 17:12:23 +0000
@@ -225,7 +225,8 @@
  This package was created automatically by linaro-media-create
 ''')
 
-    def make_package(self, name, version, relationships, architecture='all'):
+    def make_package(self, name, version, relationships, architecture='all',
+                     files=[]):
         tmp_dir = self.make_temporary_directory()
         filename = '%s_%s_%s' % (name, version, architecture)
         packaging_dir = os.path.join(tmp_dir, filename)
@@ -243,6 +244,14 @@
             )
         control_file_text = self.control_file_template.safe_substitute(
             subst_vars)
+
+        # If any files have been specified, create them
+        for file_path in files:
+            os.makedirs(os.path.join(packaging_dir,
+                                     os.path.dirname(file_path)))
+            with open(os.path.join(packaging_dir, file_path), 'w') as new_file:
+                new_file.write(name + " " + file_path)
+
         with open(os.path.join(
             packaging_dir, 'DEBIAN', 'control'), 'w') as control_file:
             control_file.write(control_file_text)

=== modified file 'linaro_image_tools/hwpack/tests/test_builder.py'
--- linaro_image_tools/hwpack/tests/test_builder.py	2012-06-13 14:53:32 +0000
+++ linaro_image_tools/hwpack/tests/test_builder.py	2012-09-04 10:42:27 +0000
@@ -31,7 +31,7 @@
     HardwarePackBuilder,
     logger as builder_logger,
     )
-from linaro_image_tools.hwpack.config import HwpackConfigError
+from linaro_image_tools.hwpack.config import HwpackConfigError, Config
 from linaro_image_tools.hwpack.hardwarepack import Metadata
 from linaro_image_tools.hwpack.packages import (
     FetchedPackage,
@@ -55,6 +55,7 @@
     MockSomethingFixture,
     MockCmdRunnerPopenFixture,
     )
+from StringIO import StringIO
 
 
 class ConfigFileMissingTests(TestCase):
@@ -82,7 +83,7 @@
         package_file_name = "package-to-unpack"
         with PackageUnpacker() as package_unpacker:
             package_unpacker.unpack_package(package_file_name)
-            package_dir = package_unpacker.tempdir
+            package_dir = package_unpacker.get_path(package_file_name)
         self.assertEquals(
             ["tar -C %s -xf -" % package_dir,
              "dpkg --fsys-tarfile %s" % package_file_name],
@@ -98,7 +99,7 @@
                     os.path, 'exists', lambda file: True))
             tempfile = package_unpacker.get_file(package, file)
             self.assertEquals(tempfile,
-                              os.path.join(package_unpacker.tempdir, file))
+                   os.path.join(package_unpacker.get_path(package), file))
 
     def test_get_file_raises(self):
         package = 'package'
@@ -109,8 +110,42 @@
             self.assertRaises(AssertionError, package_unpacker.get_file,
                               package, file)
 
+    def test_get_file_no_clash(self):
+        # Test that PackageUnpacker, asked to get the same file path
+        # from 2 different packages, return reference to *different*
+        # temporary files
+        package1 = 'package1'
+        package2 = 'package2'
+        file = 'dummyfile'
+        with PackageUnpacker() as package_unpacker:
+            self.useFixture(MockSomethingFixture(
+                    package_unpacker, 'unpack_package', lambda package: None))
+            self.useFixture(MockSomethingFixture(
+                    os.path, 'exists', lambda file: True))
+            tempfile1 = package_unpacker.get_file(package1, file)
+            tempfile2 = package_unpacker.get_file(package2, file)
+            self.assertNotEquals(tempfile1, tempfile2)
+
 
 class HardwarePackBuilderTests(TestCaseWithFixtures):
+    config_v3 = "\n".join(["format: 3.0",
+                 "name: ahwpack",
+                 "architectures: armel",
+                 "serial_tty: ttySAC1",
+                 "partition_layout:",
+                 " - bootfs_rootfs",
+                 "boot_script: boot.scr",
+                 "mmc_id: 0:1",
+                 "kernel_file: boot/vmlinuz-*-linaro-omap",
+                 "initrd_file: boot/initrd.img-*-linaro-omap",
+                 "dtb_file: boot/dt-*-linaro-omap/omap4-panda.dtb",
+                 "packages:",
+                 " - %s",
+                 " - %s",
+                 ""])
+    bootloader_config = "\n".join(["  package: %s",
+                                   "  in_boot_part: %s",
+                                   ""])
 
     def setUp(self):
         super(HardwarePackBuilderTests, self).setUp()
@@ -394,3 +429,83 @@
         self.assertThat(
             handler.messages[0].getMessage(),
             Equals("Local package 'bar' not included"))
+
+    def test_global_and_board_bootloader(self):
+        package_names = ['package0', 'package1']
+        files = {package_names[0]:
+                     ["usr/lib/u-boot/omap4_panda/u-boot.img",
+                      "usr/share/doc/u-boot-linaro-omap4-panda/copyright"],
+                 package_names[1]: ["usr/lib/u-boot/omap4_panda/u-boot.img"]}
+
+        config_v3 = self.config_v3 + "\n".join([
+            "bootloaders:",
+            " u_boot:",
+            self.bootloader_config,
+            "  file: " + files[package_names[0]][0],
+            "  copy_files:",
+            "   - " + files[package_names[0]][1],
+            "boards:",
+            " board1:",
+            "  bootloaders:",
+            "   u_boot:",
+            "    package: %s",
+            "    file: " + files[package_names[1]][0],
+            "    in_boot_part: true",
+            "sources:",
+            " ubuntu: %s"])
+
+        # Generate some test packages
+        available_packages = []
+        maker = PackageMaker()
+        self.useFixture(ContextManagerFixture(maker))
+
+        for package_name in package_names:
+            # The files parameter to make_package is a list of files to create.
+            # These files are text files containing package_name and their
+            # path. Since package_name is different for each package, this
+            # gives each file a unique content.
+            deb_file_path = maker.make_package(package_name, '1.0', {},
+                                               files=files[package_name])
+            dummy_package = DummyFetchedPackage(
+                package_name, "1.0", content=open(deb_file_path).read())
+            available_packages.append(dummy_package)
+
+        source = self.useFixture(AptSourceFixture(available_packages))
+
+        # Generate a V3 config
+        config_v3 = config_v3 % (package_names[0], package_names[1],
+                                 package_names[0], "True",
+                                 package_names[1],
+                                 source.sources_entry)
+
+        config_file_fixture = self.useFixture(ConfigFileFixture(config_v3))
+
+        # Parse the config
+        config = Config(StringIO(config_v3))
+        config.set_bootloader("u_boot")
+
+        # Build a hardware pack
+        builder = HardwarePackBuilder(
+            config_file_fixture.filename, "1.0",
+            [os.path.join(source.rootdir, package.filepath)
+             for package in available_packages])
+
+        builder.build()
+
+        # Read the contents of the hardware pack, making sure it is as expected
+        tf = tarfile.open("hwpack_ahwpack_1.0_armel.tar.gz", mode="r:gz")
+
+        # We check the content of each file when content != None. For our test
+        # files this is "<package_name> <originating path>" so they can be
+        # uniquely identified.
+        expected_files = [
+            ("u_boot/u-boot.img",
+             package_names[0] + " " + files[package_names[0]][0]),
+            ("u_boot/copyright",
+             package_names[0] + " " + files[package_names[0]][1]),
+            ("u_boot-board1/u-boot.img",
+             package_names[1] + " " + files[package_names[1]][0])]
+
+        for expected_file, contents in expected_files:
+            self.assertThat(
+                tf, TarfileHasFile(expected_file, content=contents))

