=== modified file 'linaro_image_tools/hwpack/builder.py'
@@ -41,14 +41,20 @@
PACKAGE_FIELD,
SPL_FILE_FIELD,
SPL_PACKAGE_FIELD,
+ COPY_FILES_FIELD,
)
# The fields that hold packages to be installed.
PACKAGE_FIELDS = [PACKAGE_FIELD, SPL_PACKAGE_FIELD]
-# The fields that hold values that should be reset to newly calculated ones.
-# The values of the dictionary are the fields whose values should be reset.
-FIELDS_TO_CHANGE = {PACKAGE_FIELD: FILE_FIELD,
- SPL_PACKAGE_FIELD: SPL_FILE_FIELD}
+# Specification of files (boot related) to extract:
+# <field_containing_filepaths>: (<take_files_from_package>,
+# <put_into_this_hwpack_subdir>)
+# if <put_into_this_hwpack_subdir> is None, it will be <bootloader_name> for
+# global bootloader, or <board>-<bootloader_name> for board-specific
+# bootloader
+EXTRACT_FILES = {FILE_FIELD: (PACKAGE_FIELD, None),
+ SPL_FILE_FIELD: (SPL_PACKAGE_FIELD, None),
+ COPY_FILES_FIELD: (PACKAGE_FIELD, None)}
logger = logging.getLogger(__name__)
@@ -74,19 +80,30 @@
if self.tempdir is not None and os.path.exists(self.tempdir):
shutil.rmtree(self.tempdir)
+ def get_path(self, package_file_name, file_name=''):
+ "Get package or file path in unpacker tmp dir."
+ package_dir = os.path.basename(package_file_name)
+ return os.path.join(self.tempdir, package_dir, file_name)
+
def unpack_package(self, package_file_name):
# We could extract only a single file, but since dpkg will pipe
# the entire package through tar anyway we might as well extract all.
- p = cmd_runner.run(["tar", "-C", self.tempdir, "-xf", "-"],
+ unpack_dir = self.get_path(package_file_name)
+ if not os.path.isdir(unpack_dir):
+ os.mkdir(unpack_dir)
+ p = cmd_runner.run(["tar", "-C", unpack_dir, "-xf", "-"],
stdin=subprocess.PIPE)
cmd_runner.run(["dpkg", "--fsys-tarfile", package_file_name],
stdout=p.stdin).communicate()
p.communicate()
def get_file(self, package, file):
+ # File path passed here must not be absolute, or file from
+ # real filesystem will be referenced.
+ assert file and file[0] != '/'
self.unpack_package(package)
logger.debug("Unpacked package %s." % package)
- temp_file = os.path.join(self.tempdir, file)
+ temp_file = self.get_path(package, file)
assert os.path.exists(temp_file), "The file '%s' was " \
"not found in the package '%s'." % (file, package)
return temp_file
@@ -94,7 +111,7 @@
class HardwarePackBuilder(object):
- def __init__(self, config_path, version, local_debs):
+ def __init__(self, config_path, version, local_debs, out_name=None):
try:
with open(config_path) as fp:
self.config = Config(fp, allow_unset_bootloader=True)
@@ -110,6 +127,7 @@
self.hwpack = None
self.packages = None
self.packages_added_to_hwpack = []
+ self.out_name = out_name
def find_fetched_package(self, packages, wanted_package_name):
wanted_package = None
@@ -123,7 +141,7 @@
return wanted_package
def add_file_to_hwpack(self, package, wanted_file, target_path):
- if (package.name, target_path) in self.packages_added_to_hwpack:
+ if (package.name, wanted_file) in self.packages_added_to_hwpack:
# Don't bother adding the same package more than once.
return
@@ -148,34 +166,57 @@
# Eliminate duplicates.
return list(set(boot_packages))
- def _set_new_values(self, config_dictionary):
- """Loop through the bootloaders sections of a hwpack, also from the
- boards section, changing the necessary values with the newly calculated
- ones.
-
- :param config_dictionary: The dictionary from the Config we need to
- look into.
- """
- for key, value in config_dictionary.iteritems():
- if isinstance(value, dict):
- self._set_new_values(value)
- else:
- if key in FIELDS_TO_CHANGE.keys():
- if key == PACKAGE_FIELD:
- # Need to use the correct path for the packages.
- path = self.hwpack.U_BOOT_DIR
- else:
- path = self.hwpack.SPL_DIR
- change_field = FIELDS_TO_CHANGE.get(key)
- boot_package = value
- boot_file = config_dictionary.get(change_field)
- if boot_package is not None and boot_file is not None:
- package = self.find_fetched_package(
- self.packages,
- boot_package)
- file_to_add = self.add_file_to_hwpack(
- package, boot_file, path)
- config_dictionary[change_field] = file_to_add
+ def extract_bootloader_files(self, board, bootloader_name,
+ bootloader_conf):
+ for key, value in bootloader_conf.iteritems():
+ if key in EXTRACT_FILES:
+ package_field, dest_path = EXTRACT_FILES[key]
+ if not dest_path:
+ dest_path = bootloader_name
+ if board:
+ dest_path += "-" + board
+ # Dereference package field to get actual package name
+ package = bootloader_conf.get(package_field)
+ src_files = value
+
+ # Process scalar and list fields consistently below
+ field_value_scalar = False
+ if type(src_files) != type([]):
+ src_files = [src_files]
+ field_value_scalar = True
+
+ package_ref = self.find_fetched_package(
+ self.packages, package)
+ added_files = []
+ for f in src_files:
+ added_files.append(self.add_file_to_hwpack(
+ package_ref, f, dest_path))
+ # Store within-hwpack file paths with the same
+ # scalar/list type as original field.
+ if field_value_scalar:
+ assert len(added_files) == 1
+ added_files = added_files[0]
+ bootloader_conf[key] = added_files
+
+ def extract_files(self, config_dictionary, is_bootloader_config,
+ board=None):
+ """Extract (boot) files based on EXTRACT_FILES spec and put
+ them into hwpack."""
+ self.remove_packages = []
+ if is_bootloader_config:
+ for bootl_name, bootl_conf in config_dictionary.iteritems():
+ self.extract_bootloader_files(board, bootl_name, bootl_conf)
+ else:
+ # This is board config
+ for board, board_conf in config_dictionary.iteritems():
+ bootloaders = board_conf['bootloaders']
+ self.extract_files(bootloaders, True, board)
+
+ # Clean up no longer needed packages.
+ for package in self.remove_packages:
+ if package in self.packages:
+ self.packages.remove(package)
+ self.remove_packages = []
def build(self):
for architecture in self.config.architectures:
@@ -186,7 +227,10 @@
sources = self.config.sources
with LocalArchiveMaker() as local_archive_maker:
self.hwpack.add_apt_sources(sources)
- sources = sources.values()
+ if sources:
+ sources = sources.values()
+ else:
+ sources = []
self.packages = self.config.packages[:]
# Loop through multiple bootloaders.
# In V3 of hwpack configuration, all the bootloaders info and
@@ -226,10 +270,11 @@
# through both of them changing what is necessary.
if self.config.format.format_as_string == '3.0':
if self.config.bootloaders is not None:
- self._set_new_values(self.config.bootloaders)
+ self.extract_files(self.config.bootloaders,
+ True)
metadata.bootloaders = self.config.bootloaders
if self.config.boards is not None:
- self._set_new_values(self.config.boards)
+ self.extract_files(self.config.boards, False)
metadata.boards = self.config.boards
else:
bootloader_package = None
@@ -275,9 +320,15 @@
local_package.name)
self.hwpack.add_dependency_package(
self.config.packages)
- with open(self.hwpack.filename(), 'w') as f:
+ out_name = self.out_name
+ if not out_name:
+ out_name = self.hwpack.filename()
+ with open(out_name, 'w') as f:
self.hwpack.to_file(f)
- logger.info("Wrote %s" % self.hwpack.filename())
- with open(self.hwpack.filename('.manifest.txt'),
- 'w') as f:
+ logger.info("Wrote %s" % out_name)
+ manifest_name = os.path.splitext(out_name)[0]
+ if manifest_name.endswith('.tar'):
+ manifest_name = os.path.splitext(manifest_name)[0]
+ manifest_name += '.manifest.txt'
+ with open(manifest_name, 'w') as f:
f.write(self.hwpack.manifest_text())
=== modified file 'linaro_image_tools/hwpack/hardwarepack.py'
@@ -434,6 +434,7 @@
SOURCES_LIST_GPG_DIRNAME = "sources.list.d.gpg"
U_BOOT_DIR = "u-boot"
SPL_DIR = "spl"
+ BOOT_DIR = "boot"
def __init__(self, metadata):
"""Create a HardwarePack.
@@ -482,7 +483,8 @@
to sources entries.
:type sources: a dict mapping str to str
"""
- self.sources.update(sources)
+ if sources:
+ self.sources.update(sources)
def add_packages(self, packages):
"""Add packages to the hardware pack.
=== modified file 'linaro_image_tools/hwpack/packages.py'
@@ -225,7 +225,8 @@
This package was created automatically by linaro-media-create
''')
- def make_package(self, name, version, relationships, architecture='all'):
+ def make_package(self, name, version, relationships, architecture='all',
+ files=[]):
tmp_dir = self.make_temporary_directory()
filename = '%s_%s_%s' % (name, version, architecture)
packaging_dir = os.path.join(tmp_dir, filename)
@@ -243,6 +244,14 @@
)
control_file_text = self.control_file_template.safe_substitute(
subst_vars)
+
+ # If any files have been specified, create them
+ for file_path in files:
+ os.makedirs(os.path.join(packaging_dir,
+ os.path.dirname(file_path)))
+ with open(os.path.join(packaging_dir, file_path), 'w') as new_file:
+ new_file.write(name + " " + file_path)
+
with open(os.path.join(
packaging_dir, 'DEBIAN', 'control'), 'w') as control_file:
control_file.write(control_file_text)
=== modified file 'linaro_image_tools/hwpack/tests/test_builder.py'
@@ -31,7 +31,7 @@
HardwarePackBuilder,
logger as builder_logger,
)
-from linaro_image_tools.hwpack.config import HwpackConfigError
+from linaro_image_tools.hwpack.config import HwpackConfigError, Config
from linaro_image_tools.hwpack.hardwarepack import Metadata
from linaro_image_tools.hwpack.packages import (
FetchedPackage,
@@ -55,6 +55,7 @@
MockSomethingFixture,
MockCmdRunnerPopenFixture,
)
+from StringIO import StringIO
class ConfigFileMissingTests(TestCase):
@@ -82,7 +83,7 @@
package_file_name = "package-to-unpack"
with PackageUnpacker() as package_unpacker:
package_unpacker.unpack_package(package_file_name)
- package_dir = package_unpacker.tempdir
+ package_dir = package_unpacker.get_path(package_file_name)
self.assertEquals(
["tar -C %s -xf -" % package_dir,
"dpkg --fsys-tarfile %s" % package_file_name],
@@ -98,7 +99,7 @@
os.path, 'exists', lambda file: True))
tempfile = package_unpacker.get_file(package, file)
self.assertEquals(tempfile,
- os.path.join(package_unpacker.tempdir, file))
+ os.path.join(package_unpacker.get_path(package), file))
def test_get_file_raises(self):
package = 'package'
@@ -109,8 +110,42 @@
self.assertRaises(AssertionError, package_unpacker.get_file,
package, file)
+ def test_get_file_no_clash(self):
+ # Test that PackageUnpacker, asked to get the same file path
+ # from 2 different packages, return reference to *different*
+ # temporary files
+ package1 = 'package1'
+ package2 = 'package2'
+ file = 'dummyfile'
+ with PackageUnpacker() as package_unpacker:
+ self.useFixture(MockSomethingFixture(
+ package_unpacker, 'unpack_package', lambda package: None))
+ self.useFixture(MockSomethingFixture(
+ os.path, 'exists', lambda file: True))
+ tempfile1 = package_unpacker.get_file(package1, file)
+ tempfile2 = package_unpacker.get_file(package2, file)
+ self.assertNotEquals(tempfile1, tempfile2)
+
class HardwarePackBuilderTests(TestCaseWithFixtures):
+ config_v3 = "\n".join(["format: 3.0",
+ "name: ahwpack",
+ "architectures: armel",
+ "serial_tty: ttySAC1",
+ "partition_layout:",
+ " - bootfs_rootfs",
+ "boot_script: boot.scr",
+ "mmc_id: 0:1",
+ "kernel_file: boot/vmlinuz-*-linaro-omap",
+ "initrd_file: boot/initrd.img-*-linaro-omap",
+ "dtb_file: boot/dt-*-linaro-omap/omap4-panda.dtb",
+ "packages:",
+ " - %s",
+ " - %s",
+ ""])
+ bootloader_config = "\n".join([" package: %s",
+ " in_boot_part: %s",
+ ""])
def setUp(self):
super(HardwarePackBuilderTests, self).setUp()
@@ -394,3 +429,83 @@
self.assertThat(
handler.messages[0].getMessage(),
Equals("Local package 'bar' not included"))
+
+ def test_global_and_board_bootloader(self):
+ package_names = ['package0', 'package1']
+ files = {package_names[0]:
+ ["usr/lib/u-boot/omap4_panda/u-boot.img",
+ "usr/share/doc/u-boot-linaro-omap4-panda/copyright"],
+ package_names[1]: ["usr/lib/u-boot/omap4_panda/u-boot.img"]}
+
+ config_v3 = self.config_v3 + "\n".join([
+ "bootloaders:",
+ " u_boot:",
+ self.bootloader_config,
+ " file: " + files[package_names[0]][0],
+ " copy_files:",
+ " - " + files[package_names[0]][1],
+ "boards:",
+ " board1:",
+ " bootloaders:",
+ " u_boot:",
+ " package: %s",
+ " file: " + files[package_names[1]][0],
+ " in_boot_part: true",
+ "sources:",
+ " ubuntu: %s"])
+
+ # Generate some test packages
+ available_packages = []
+ maker = PackageMaker()
+ self.useFixture(ContextManagerFixture(maker))
+
+ for package_name in package_names:
+ # The files parameter to make_package is a list of files to create.
+ # These files are text files containing package_name and their
+ # path. Since package_name is different for each package, this
+ # gives each file a unique content.
+ deb_file_path = maker.make_package(package_name, '1.0', {},
+ files=files[package_name])
+ dummy_package = DummyFetchedPackage(
+ package_name, "1.0", content=open(deb_file_path).read())
+ available_packages.append(dummy_package)
+
+ source = self.useFixture(AptSourceFixture(available_packages))
+
+ # Generate a V3 config
+ config_v3 = config_v3 % (package_names[0], package_names[1],
+ package_names[0], "True",
+ package_names[1],
+ source.sources_entry)
+
+ config_file_fixture = self.useFixture(ConfigFileFixture(config_v3))
+
+ # Parse the config
+ config = Config(StringIO(config_v3))
+ config.set_bootloader("u_boot")
+
+ # Build a hardware pack
+ builder = HardwarePackBuilder(
+ config_file_fixture.filename, "1.0",
+ [os.path.join(source.rootdir, package.filepath)
+ for package in available_packages])
+
+ builder.build()
+
+ # Read the contents of the hardware pack, making sure it is as expected
+ tf = tarfile.open("hwpack_ahwpack_1.0_armel.tar.gz", mode="r:gz")
+
+ # We check the content of each file when content != None. For our test
+ # files this is "<package_name> <originating path>" so they can be
+ # uniquely identified.
+ expected_files = [
+ ("u_boot/u-boot.img",
+ package_names[0] + " " + files[package_names[0]][0]),
+ ("u_boot/copyright",
+ package_names[0] + " " + files[package_names[0]][1]),
+ ("u_boot-board1/u-boot.img",
+ package_names[1] + " " + files[package_names[1]][0])]
+
+ for expected_file, contents in expected_files:
+ self.assertThat(
+ tf, TarfileHasFile(expected_file, content=contents))