From patchwork Tue Jun 21 09:56:22 2011 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mattias Backman X-Patchwork-Id: 2115 Return-Path: X-Original-To: patchwork@peony.canonical.com Delivered-To: patchwork@peony.canonical.com Received: from fiordland.canonical.com (fiordland.canonical.com [91.189.94.145]) by peony.canonical.com (Postfix) with ESMTP id 1F13323E02 for ; Tue, 21 Jun 2011 09:56:27 +0000 (UTC) Received: from mail-vw0-f52.google.com (mail-vw0-f52.google.com [209.85.212.52]) by fiordland.canonical.com (Postfix) with ESMTP id 7EA6AA187CE for ; Tue, 21 Jun 2011 09:56:26 +0000 (UTC) Received: by vws16 with SMTP id 16so4490656vws.11 for ; Tue, 21 Jun 2011 02:56:25 -0700 (PDT) Received: by 10.52.98.97 with SMTP id eh1mr8882870vdb.7.1308650185128; Tue, 21 Jun 2011 02:56:25 -0700 (PDT) X-Forwarded-To: linaro-patchwork@canonical.com X-Forwarded-For: patch@linaro.org linaro-patchwork@canonical.com Delivered-To: patches@linaro.org Received: by 10.52.183.130 with SMTP id em2cs55978vdc; Tue, 21 Jun 2011 02:56:24 -0700 (PDT) Received: by 10.227.39.88 with SMTP id f24mr3396044wbe.88.1308650183398; Tue, 21 Jun 2011 02:56:23 -0700 (PDT) Received: from adelie.canonical.com (adelie.canonical.com [91.189.90.139]) by mx.google.com with ESMTP id fr1si11270016wbb.110.2011.06.21.02.56.22; Tue, 21 Jun 2011 02:56:23 -0700 (PDT) Received-SPF: pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.139 as permitted sender) client-ip=91.189.90.139; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.139 as permitted sender) smtp.mail=bounces@canonical.com Received: from loganberry.canonical.com ([91.189.90.37]) by adelie.canonical.com with esmtp (Exim 4.71 #1 (Debian)) id 1QYxh0-0007kz-HF for ; Tue, 21 Jun 2011 09:56:22 +0000 Received: from loganberry.canonical.com (localhost [127.0.0.1]) by loganberry.canonical.com (Postfix) with ESMTP id 7F0712E84FB for ; Tue, 21 Jun 2011 09:56:22 +0000 (UTC) MIME-Version: 1.0 X-Launchpad-Project: linaro-image-tools X-Launchpad-Branch: ~linaro-image-tools/linaro-image-tools/trunk X-Launchpad-Message-Rationale: Subscriber X-Launchpad-Branch-Revision-Number: 359 X-Launchpad-Notification-Type: branch-revision To: Linaro Patch Tracker From: noreply@launchpad.net Subject: [Branch ~linaro-image-tools/linaro-image-tools/trunk] Rev 359: Merge lp:~mabac/linaro-image-tools/hwpacks-v2-hwpack-create; add support for hwpacks V2 format to... Message-Id: <20110621095622.24394.40958.launchpad@loganberry.canonical.com> Date: Tue, 21 Jun 2011 09:56:22 -0000 Reply-To: noreply@launchpad.net Sender: bounces@canonical.com Errors-To: bounces@canonical.com Precedence: bulk X-Generated-By: Launchpad (canonical.com); Revision="13242"; Instance="initZopeless config overlay" X-Launchpad-Hash: d155e39c90ad06ac692abe8ab74823fc180b7598 Merge authors: Mattias Backman (mabac) Related merge proposals: https://code.launchpad.net/~mabac/linaro-image-tools/hwpacks-v2-hwpack-create/+merge/60177 proposed by: Mattias Backman (mabac) review: Approve - James Westby (james-w) ------------------------------------------------------------ revno: 359 [merge] committer: Mattias Backman branch nick: linaro-image-tools timestamp: Tue 2011-06-21 11:45:11 +0200 message: Merge lp:~mabac/linaro-image-tools/hwpacks-v2-hwpack-create; add support for hwpacks V2 format to linaro-hwpack-create. added: linaro_image_tools/hwpack/hardwarepack_format.py modified: linaro_image_tools/hwpack/builder.py linaro_image_tools/hwpack/config.py linaro_image_tools/hwpack/hardwarepack.py linaro_image_tools/hwpack/packages.py linaro_image_tools/hwpack/tests/test_builder.py linaro_image_tools/hwpack/tests/test_config.py linaro_image_tools/hwpack/tests/test_hardwarepack.py linaro_image_tools/tests/fixtures.py --- lp:linaro-image-tools https://code.launchpad.net/~linaro-image-tools/linaro-image-tools/trunk You are subscribed to branch lp:linaro-image-tools. To unsubscribe from this branch go to https://code.launchpad.net/~linaro-image-tools/linaro-image-tools/trunk/+edit-subscription === modified file 'linaro_image_tools/hwpack/builder.py' --- linaro_image_tools/hwpack/builder.py 2011-03-23 22:25:10 +0000 +++ linaro_image_tools/hwpack/builder.py 2011-06-20 13:33:01 +0000 @@ -21,6 +21,12 @@ import logging import errno +import subprocess +import tempfile +import os +import shutil + +from linaro_image_tools import cmd_runner from linaro_image_tools.hwpack.config import Config from linaro_image_tools.hwpack.hardwarepack import HardwarePack, Metadata @@ -45,6 +51,33 @@ "No such config file: '%s'" % self.filename) +class PackageUnpacker(object): + def __enter__(self): + self.tempdir = tempfile.mkdtemp() + return self + + def __exit__(self, type, value, traceback): + if self.tempdir is not None and os.path.exists(self.tempdir): + shutil.rmtree(self.tempdir) + + def unpack_package(self, package_file_name): + # We could extract only a single file, but since dpkg will pipe + # the entire package through tar anyway we might as well extract all. + p = cmd_runner.run(["tar", "-C", self.tempdir, "-xf", "-"], + stdin=subprocess.PIPE) + cmd_runner.run(["dpkg", "--fsys-tarfile", package_file_name], + stdout=p.stdin).communicate() + p.communicate() + + def get_file(self, package, file): + self.unpack_package(package) + logger.debug("Unpacked package %s." % package) + temp_file = os.path.join(self.tempdir, file) + assert os.path.exists(temp_file), "The file '%s' was " \ + "not found in the package '%s'." % (file, package) + return temp_file + + class HardwarePackBuilder(object): def __init__(self, config_path, version, local_debs): @@ -56,9 +89,27 @@ raise ConfigFileMissing(config_path) raise self.config.validate() + self.format = self.config.format self.version = version self.local_debs = local_debs + def find_fetched_package(self, packages, wanted_package_name): + wanted_package = None + for package in packages: + if package.name == wanted_package_name: + wanted_package = package + break + else: + raise AssertionError("Package '%s' was not fetched." % \ + wanted_package_name) + packages.remove(wanted_package) + return wanted_package + + def add_file_to_hwpack(self, package, wanted_file, package_unpacker, hwpack, target_path): + tempfile_name = package_unpacker.get_file( + package.filepath, wanted_file) + return hwpack.add_file(target_path, tempfile_name) + def build(self): for architecture in self.config.architectures: logger.info("Building for %s" % architecture) @@ -70,6 +121,8 @@ hwpack.add_apt_sources(sources) sources = sources.values() packages = self.config.packages[:] + if self.config.u_boot_package is not None: + packages.append(self.config.u_boot_package) local_packages = [ FetchedPackage.from_deb(deb) for deb in self.local_debs] @@ -81,10 +134,18 @@ fetcher = PackageFetcher( sources, architecture=architecture, prefer_label=LOCAL_ARCHIVE_LABEL) - with fetcher: + with fetcher, PackageUnpacker() as package_unpacker: fetcher.ignore_packages(self.config.assume_installed) packages = fetcher.fetch_packages( packages, download_content=self.config.include_debs) + + if self.config.u_boot_package is not None: + u_boot_package = self.find_fetched_package( + packages, self.config.u_boot_package) + hwpack.metadata.u_boot = self.add_file_to_hwpack( + u_boot_package, self.config.u_boot_file, + package_unpacker, hwpack, hwpack.U_BOOT_DIR) + logger.debug("Adding packages to hwpack") hwpack.add_packages(packages) for local_package in local_packages: === modified file 'linaro_image_tools/hwpack/config.py' --- linaro_image_tools/hwpack/config.py 2011-01-28 19:50:48 +0000 +++ linaro_image_tools/hwpack/config.py 2011-06-20 13:46:16 +0000 @@ -22,6 +22,10 @@ import ConfigParser import re +from hardwarepack_format import ( + HardwarePackFormatV1, + HardwarePackFormatV2, + ) class HwpackConfigError(Exception): pass @@ -38,10 +42,22 @@ SOURCES_ENTRY_KEY = "sources-entry" PACKAGES_KEY = "packages" PACKAGE_REGEX = NAME_REGEX + PATH_REGEX = r"[a-z0-9][a-z0-9+\-./]+$" ORIGIN_KEY = "origin" MAINTAINER_KEY = "maintainer" ARCHITECTURES_KEY = "architectures" ASSUME_INSTALLED_KEY = "assume-installed" + U_BOOT_PACKAGE_KEY = "u-boot-package" + U_BOOT_FILE_KEY = "u-boot-file" + SERIAL_TTY_KEY = "serial_tty" + KERNEL_ADDR_KEY = "kernel_addr" + INITRD_ADDR_KEY = "initrd_addr" + LOAD_ADDR_KEY = "load_addr" + WIRED_INTERFACES_KEY = "wired_interfaces" + WIRELESS_INTERFACES_KEY = "wireless_interfaces" + PARTITION_LAYOUT_KEY = "partition_layout" + MMC_ID_KEY = "mmc_id" + FORMAT_KEY = "format" def __init__(self, fp): """Create a Config. @@ -58,15 +74,49 @@ """ if not self.parser.has_section(self.MAIN_SECTION): raise HwpackConfigError("No [%s] section" % self.MAIN_SECTION) + self._validate_format() self._validate_name() self._validate_include_debs() self._validate_support() self._validate_packages() self._validate_architectures() self._validate_assume_installed() + + if self.format.has_v2_fields: + self._validate_u_boot_package() + self._validate_u_boot_file() + self._validate_serial_tty() + self._validate_kernel_addr() + self._validate_initrd_addr() + self._validate_load_addr() + self._validate_wired_interfaces() + self._validate_wireless_interfaces() + self._validate_partition_layout() + self._validate_mmc_id() + self._validate_sections() @property + def format(self): + """The format of the hardware pack. A subclass of HardwarePackFormat. + """ + try: + format_string = self.parser.get(self.MAIN_SECTION, self.FORMAT_KEY) + except ConfigParser.NoOptionError: + # Default to 1.0 to aviod breaking existing hwpack files. + # When this code no longer supports 1.0, it effectively makes + # explicitly specifying format in hwpack files mandatory. + format_string = "1.0" + + if format_string == '1.0': + return HardwarePackFormatV1() + elif format_string == '2.0': + return HardwarePackFormatV2() + else: + raise HwpackConfigError("Format version '%s' is not supported." % \ + format_string) + + @property def name(self): """The name of the hardware pack. A str.""" return self.parser.get(self.MAIN_SECTION, self.NAME_KEY) @@ -101,6 +151,72 @@ return None @property + def serial_tty(self): + """/dev device name of the serial console for this kernel + + A str. + """ + return self._get_option_from_main_section(self.SERIAL_TTY_KEY) + + @property + def kernel_addr(self): + """address where u-boot should load the kernel + + An int. + """ + return self._get_option_from_main_section(self.KERNEL_ADDR_KEY) + + @property + def initrd_addr(self): + """address where u-boot should load the kernel + + An int. + """ + return self._get_option_from_main_section(self.INITRD_ADDR_KEY) + + @property + def load_addr(self): + """address for uImage generation + + An int. + """ + return self._get_option_from_main_section(self.LOAD_ADDR_KEY) + + @property + def wired_interfaces(self): + """The interfaces for wired networks + + A list of str. + """ + return self._get_list_from_main_section(self.WIRED_INTERFACES_KEY) + + @property + def wireless_interfaces(self): + """The interfaces for wireless networks + + A list of str. + """ + return self._get_list_from_main_section(self.WIRELESS_INTERFACES_KEY) + + @property + def partition_layout(self): + """bootfs16_rootfs, bootfs_rootfs and reserved_bootfs_rootfs; + controls what kind of SD card partition layout we should use when + writing images + + A str. + """ + return self._get_option_from_main_section(self.PARTITION_LAYOUT_KEY) + + @property + def mmc_id(self): + """which MMC drive contains the boot filesystem + + An int. + """ + return self._get_option_from_main_section(self.MMC_ID_KEY) + + @property def origin(self): """The origin that should be recorded in the hwpack. @@ -144,6 +260,22 @@ return self._get_list_from_main_section(self.PACKAGES_KEY) @property + def u_boot_package(self): + """The u-boot package that contains the u-boot bin. + + A str. + """ + return self._get_option_from_main_section(self.U_BOOT_PACKAGE_KEY) + + @property + def u_boot_file(self): + """The u-boot bin file that will be unpacked from the u-boot package. + + A str. + """ + return self._get_option_from_main_section(self.U_BOOT_FILE_KEY) + + @property def architectures(self): """The architectures to build the hwpack for. @@ -174,17 +306,96 @@ section_name, self.SOURCES_ENTRY_KEY) return sources + def _validate_format(self): + format = self.format + if not format: + raise HwpackConfigError("Empty value for format") + if not format.is_supported: + raise HwpackConfigError("Format version '%s' is not supported." % \ + format) + + def _assert_matches_pattern(self, regex, config_item, error_message): + if re.match(regex, config_item) is None: + raise HwpackConfigError(error_message) + def _validate_name(self): try: name = self.name if not name: raise HwpackConfigError("Empty value for name") - if re.match(self.NAME_REGEX, name) is None: - raise HwpackConfigError("Invalid name: %s" % name) + self._assert_matches_pattern( + self.NAME_REGEX, name, "Invalid name: %s" % name) except ConfigParser.NoOptionError: raise HwpackConfigError( "No name in the [%s] section" % self.MAIN_SECTION) + def _validate_u_boot_file(self): + u_boot_file = self.u_boot_file + if not u_boot_file: + raise HwpackConfigError("No u_boot_file in the [%s] section" % \ + self.MAIN_SECTION) + self._assert_matches_pattern( + self.PATH_REGEX, u_boot_file, "Invalid path: %s" % u_boot_file) + + def _validate_serial_tty(self): + serial_tty = self.serial_tty + if serial_tty is None: + return + if len(serial_tty) < 4 or serial_tty[:3] != 'tty': + raise HwpackConfigError("Invalid serial tty: %s" % serial_tty) + + def _validate_addr(self, addr): + return re.match(r"^0x[a-fA-F0-9]{8}$", addr) + + def _validate_kernel_addr(self): + addr = self.kernel_addr + if addr is None: + return + if not self._validate_addr(addr): + raise HwpackConfigError("Invalid kernel address: %s" % addr) + + def _validate_initrd_addr(self): + addr = self.initrd_addr + if addr is None: + return + if not self._validate_addr(addr): + raise HwpackConfigError("Invalid initrd address: %s" % addr) + + def _validate_load_addr(self): + addr = self.load_addr + if addr is None: + return + if not self._validate_addr(addr): + raise HwpackConfigError("Invalid load address: %s" % addr) + + def _validate_wired_interfaces(self): + pass + + def _validate_wireless_interfaces(self): + pass + + def _validate_partition_layout(self): + defined_partition_layouts = [ + #'bootfs16_rootfs', + 'bootfs_rootfs', + #'reserved_bootfs_rootfs', + ] + if self.partition_layout not in defined_partition_layouts: + raise HwpackConfigError( + "Undefined partition layout %s in the [%s] section. " + "Valid partition layouts are %s." + % (self.partition_layout, self.MAIN_SECTION, + ", ".join(defined_partition_layouts))) + + def _validate_mmc_id(self): + mmc_id = self.mmc_id + if mmc_id is None: + return + try: + int(mmc_id) + except: + raise HwpackConfigError("Invalid mmc id %s" % (mmc_id)) + def _validate_include_debs(self): try: self.include_debs @@ -206,10 +417,21 @@ "No %s in the [%s] section" % (self.PACKAGES_KEY, self.MAIN_SECTION)) for package in packages: - if re.match(self.PACKAGE_REGEX, package) is None: - raise HwpackConfigError( - "Invalid value in %s in the [%s] section: %s" - % (self.PACKAGES_KEY, self.MAIN_SECTION, package)) + self._assert_matches_pattern( + self.PACKAGE_REGEX, package, "Invalid value in %s in the " \ + "[%s] section: %s" % (self.PACKAGES_KEY, self.MAIN_SECTION, + package)) + + def _validate_u_boot_package(self): + u_boot_package = self.u_boot_package + if not u_boot_package: + raise HwpackConfigError( + "No %s in the [%s] section" + % (self.U_BOOT_PACKAGE_KEY, self.MAIN_SECTION)) + self._assert_matches_pattern( + self.PACKAGE_REGEX, u_boot_package, "Invalid value in %s in the " \ + "[%s] section: %s" % (self.U_BOOT_PACKAGE_KEY, + self.MAIN_SECTION, u_boot_package)) def _validate_architectures(self): architectures = self.architectures @@ -221,11 +443,10 @@ def _validate_assume_installed(self): assume_installed = self.assume_installed for package in assume_installed: - if re.match(self.PACKAGE_REGEX, package) is None: - raise HwpackConfigError( - "Invalid value in %s in the [%s] section: %s" - % (self.ASSUME_INSTALLED_KEY, self.MAIN_SECTION, - package)) + self._assert_matches_pattern( + self.PACKAGE_REGEX, package, "Invalid value in %s in the " \ + "[%s] section: %s" % (self.ASSUME_INSTALLED_KEY, + self.MAIN_SECTION, package)) def _validate_section_sources_entry(self, section_name): try: === modified file 'linaro_image_tools/hwpack/hardwarepack.py' --- linaro_image_tools/hwpack/hardwarepack.py 2011-03-23 22:25:10 +0000 +++ linaro_image_tools/hwpack/hardwarepack.py 2011-06-20 13:46:16 +0000 @@ -20,6 +20,7 @@ # USA. import time +import os from linaro_image_tools.hwpack.better_tarfile import writeable_tarfile from linaro_image_tools.hwpack.packages import ( @@ -27,6 +28,9 @@ get_packages_file, PackageMaker, ) +from linaro_image_tools.hwpack.hardwarepack_format import ( + HardwarePackFormatV1, +) class Metadata(object): @@ -55,11 +59,12 @@ """ def __init__(self, name, version, architecture, origin=None, - maintainer=None, support=None): + maintainer=None, support=None, format=HardwarePackFormatV1()): """Create the Metadata for a hardware pack. See the instance variables for a description of the arguments. """ + self.format = format self.name = name if ' ' in version: raise AssertionError( @@ -72,6 +77,25 @@ self.architecture = architecture @classmethod + def add_v2_config(self, serial_tty=None, kernel_addr=None, initrd_addr=None, + load_addr=None, fdt=None, wired_interfaces=[], + wireless_interfaces=[], partition_layout=None, + mmc_id=None): + """Add fields that are specific to the new format. + + These fields are not present in earlier config files. + """ + self.u_boot = None + self.serial_tty = serial_tty + self.kernel_addr = kernel_addr + self.initrd_addr = initrd_addr + self.load_addr = load_addr + self.wired_interfaces = wired_interfaces + self.wireless_interfaces = wireless_interfaces + self.partition_layout = partition_layout + self.mmc_id = mmc_id + + @classmethod def from_config(cls, config, version, architecture): """Create a Metadata from a Config object. @@ -89,9 +113,21 @@ targetting. :type architecture: str """ - return cls( + metadata = cls( config.name, version, architecture, origin=config.origin, - maintainer=config.maintainer, support=config.support) + maintainer=config.maintainer, support=config.support, + format=config.format) + + if config.format.has_v2_fields: + metadata.add_v2_config(serial_tty=config.serial_tty, + kernel_addr=config.kernel_addr, + initrd_addr=config.initrd_addr, + load_addr=config.load_addr, + wired_interfaces=config.wired_interfaces, + wireless_interfaces=config.wireless_interfaces, + partition_layout=config.partition_layout, + mmc_id=config.mmc_id) + return metadata def __str__(self): """Get the contents of the metadata file.""" @@ -104,6 +140,29 @@ metadata += "MAINTAINER=%s\n" % self.maintainer if self.support is not None: metadata += "SUPPORT=%s\n" % self.support + + if not self.format.has_v2_fields: + return metadata + + if self.u_boot is not None: + metadata += "U_BOOT=%s\n" % self.u_boot + if self.serial_tty is not None: + metadata += "SERIAL_TTY=%s\n" % self.serial_tty + if self.kernel_addr is not None: + metadata += "KERNEL_ADDR=%s\n" % self.kernel_addr + if self.initrd_addr is not None: + metadata += "INITRD_ADDR=%s\n" % self.initrd_addr + if self.load_addr is not None: + metadata += "LOAD_ADDR=%s\n" % self.load_addr + if self.wired_interfaces != []: + metadata += "WIRED_INTERFACES=%s\n" % " ".join(self.wired_interfaces) + if self.wireless_interfaces != []: + metadata += "WIRELESS_INTERFACES=%s\n" % " ".join( + self.wireless_interfaces) + if self.partition_layout is not None: + metadata += "PARTITION_LAYOUT=%s\n" % self.partition_layout + if self.mmc_id is not None: + metadata += "MMC_ID=%s\n" % self.mmc_id return metadata @@ -116,8 +175,6 @@ :type FORMAT: str """ - # The format version cannot contain white spaces. - FORMAT = "1.0" FORMAT_FILENAME = "FORMAT" METADATA_FILENAME = "metadata" MANIFEST_FILENAME = "manifest" @@ -125,6 +182,7 @@ PACKAGES_FILENAME = "%s/Packages" % PACKAGES_DIRNAME SOURCES_LIST_DIRNAME = "sources.list.d" SOURCES_LIST_GPG_DIRNAME = "sources.list.d.gpg" + U_BOOT_DIR = "u-boot" def __init__(self, metadata): """Create a HardwarePack. @@ -135,6 +193,8 @@ self.metadata = metadata self.sources = {} self.packages = [] + self.format = metadata.format + self.files = [] def filename(self, extension=".tar.gz"): """The filename that this hardware pack should have. @@ -200,6 +260,11 @@ relationships, self.metadata.architecture) self.packages.append(FetchedPackage.from_deb(deb_file_path)) + def add_file(self, dir, file): + target_file = os.path.join(dir, os.path.basename(file)) + self.files.append((file, target_file)) + return target_file + def manifest_text(self): manifest_content = "" for package in self.packages: @@ -225,9 +290,11 @@ kwargs["default_mtime"] = time.time() with writeable_tarfile(fileobj, mode="w:gz", **kwargs) as tf: tf.create_file_from_string( - self.FORMAT_FILENAME, self.FORMAT + "\n") + self.FORMAT_FILENAME, "%s\n" % self.format) tf.create_file_from_string( self.METADATA_FILENAME, str(self.metadata)) + for fs_file_name, arc_file_name in self.files: + tf.add(fs_file_name, arcname=arc_file_name) tf.create_dir(self.PACKAGES_DIRNAME) for package in self.packages: if package.content is not None: === added file 'linaro_image_tools/hwpack/hardwarepack_format.py' --- linaro_image_tools/hwpack/hardwarepack_format.py 1970-01-01 00:00:00 +0000 +++ linaro_image_tools/hwpack/hardwarepack_format.py 2011-06-17 12:28:43 +0000 @@ -0,0 +1,59 @@ +# Copyright (C) 2010, 2011 Linaro +# +# Author: James Westby +# +# This file is part of Linaro Image Tools. +# +# Linaro Image Tools is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# Linaro Image Tools is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Linaro Image Tools; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +import logging + + +logger = logging.getLogger(__name__) + + +class HardwarePackFormat(object): + def __init__(self): + self.format_as_string = None + self.is_deprecated = False + self.is_supported = False + self.has_v2_fields = False + + def __str__(self): + if self.format_as_string is None: + raise NotImplementedError() + if self.is_deprecated: + logger.warning("The format '%s' is deprecated, please update " \ + "your hardware pack configuration." % \ + self.format_as_string) + return self.format_as_string + + +class HardwarePackFormatV1(HardwarePackFormat): + def __init__(self): + super(HardwarePackFormatV1, self).__init__() + self.format_as_string = "1.0" + self.is_supported = True + self.is_deprecated = False + + +class HardwarePackFormatV2(HardwarePackFormat): + def __init__(self): + super(HardwarePackFormatV2, self).__init__() + self.format_as_string = "2.0" + self.is_supported = True + self.is_deprecated = False + self.has_v2_fields = True === modified file 'linaro_image_tools/hwpack/packages.py' --- linaro_image_tools/hwpack/packages.py 2011-05-25 20:59:19 +0000 +++ linaro_image_tools/hwpack/packages.py 2011-06-20 13:33:01 +0000 @@ -724,4 +724,5 @@ "The item %r could not be fetched: %s" % (acqfile.destfile, acqfile.error_text)) result_package.content = open(destfile) + result_package._file_path = destfile return fetched.values() === modified file 'linaro_image_tools/hwpack/tests/test_builder.py' --- linaro_image_tools/hwpack/tests/test_builder.py 2011-03-24 11:14:44 +0000 +++ linaro_image_tools/hwpack/tests/test_builder.py 2011-06-20 13:46:16 +0000 @@ -27,6 +27,7 @@ from linaro_image_tools.hwpack.builder import ( ConfigFileMissing, + PackageUnpacker, HardwarePackBuilder, logger as builder_logger, ) @@ -50,6 +51,10 @@ Not, ) from linaro_image_tools.testing import TestCaseWithFixtures +from linaro_image_tools.tests.fixtures import ( + MockSomethingFixture, + MockCmdRunnerPopenFixture, + ) class ConfigFileMissingTests(TestCase): @@ -59,6 +64,52 @@ self.assertEqual("No such config file: 'path'", str(exc)) +class PackageUnpackerTests(TestCaseWithFixtures): + + def test_creates_tempdir(self): + with PackageUnpacker() as package_unpacker: + self.assertTrue(os.path.exists(package_unpacker.tempdir)) + + def test_tempfiles_are_removed(self): + tempdir = None + with PackageUnpacker() as package_unpacker: + tempdir = package_unpacker.tempdir + self.assertFalse(os.path.exists(tempdir)) + + def test_unpack_package(self): + fixture = MockCmdRunnerPopenFixture(assert_child_finished=False) + self.useFixture(fixture) + package_file_name = "package-to-unpack" + with PackageUnpacker() as package_unpacker: + package_unpacker.unpack_package(package_file_name) + package_dir = package_unpacker.tempdir + self.assertEquals( + ["tar -C %s -xf -" % package_dir, + "dpkg --fsys-tarfile %s" % package_file_name], + fixture.mock.commands_executed) + + def test_get_file_returns_tempfile(self): + package = 'package' + file = 'dummyfile' + with PackageUnpacker() as package_unpacker: + self.useFixture(MockSomethingFixture( + package_unpacker, 'unpack_package', lambda package: None)) + self.useFixture(MockSomethingFixture( + os.path, 'exists', lambda file: True)) + tempfile = package_unpacker.get_file(package, file) + self.assertEquals(tempfile, + os.path.join(package_unpacker.tempdir, file)) + + def test_get_file_raises(self): + package = 'package' + file = 'dummyfile' + with PackageUnpacker() as package_unpacker: + self.useFixture(MockSomethingFixture( + package_unpacker, 'unpack_package', lambda package: None)) + self.assertRaises(AssertionError, package_unpacker.get_file, + package, file) + + class HardwarePackBuilderTests(TestCaseWithFixtures): def setUp(self): @@ -95,6 +146,58 @@ config = self.useFixture(ConfigFileFixture(config_text)) return Metadata(hwpack_name, hwpack_version, architecture), config + def test_find_fetched_package_finds(self): + package_name = "dummy-package" + wanted_package_name = "wanted-package" + available_package = DummyFetchedPackage(package_name, "1.1") + wanted_package = DummyFetchedPackage(wanted_package_name, "1.1") + + sources_dict = self.sourcesDictForPackages([available_package, + wanted_package]) + _, config = self.makeMetaDataAndConfigFixture( + [package_name, wanted_package_name], sources_dict, + extra_config={'format': '2.0', 'u-boot-package': 'wanted-package', + 'u-boot-file': 'wanted-file', + 'partition_layout': 'bootfs_rootfs'}) + builder = HardwarePackBuilder(config.filename, "1.0", []) + found_package = builder.find_fetched_package( + [available_package, wanted_package], wanted_package_name) + self.assertEquals(wanted_package, found_package) + + def test_find_fetched_package_removes(self): + package_name = "dummy-package" + wanted_package_name = "wanted-package" + available_package = DummyFetchedPackage(package_name, "1.1") + wanted_package = DummyFetchedPackage(wanted_package_name, "1.1") + + sources_dict = self.sourcesDictForPackages([available_package, + wanted_package]) + _, config = self.makeMetaDataAndConfigFixture( + [package_name, wanted_package_name], sources_dict, + extra_config={'format': '2.0', 'u-boot-package': 'wanted-package', + 'u-boot-file': 'wanted-file', + 'partition_layout': 'bootfs_rootfs'}) + builder = HardwarePackBuilder(config.filename, "1.0", []) + packages = [available_package, wanted_package] + builder.find_fetched_package(packages, wanted_package_name) + self.assertEquals(packages, [available_package]) + + def test_find_fetched_package_raises(self): + package_name = "dummy-package" + wanted_package_name = "wanted-package" + available_package = DummyFetchedPackage(package_name, "1.1") + + sources_dict = self.sourcesDictForPackages([available_package]) + _, config = self.makeMetaDataAndConfigFixture( + [package_name], sources_dict, + extra_config={'format': '2.0', 'u-boot-package': 'wanted-package', + 'u-boot-file': 'wanted-file', + 'partition_layout': 'bootfs_rootfs'}) + builder = HardwarePackBuilder(config.filename, "1.0", []) + packages = [available_package] + self.assertRaises(AssertionError, builder.find_fetched_package, + packages, wanted_package_name) + def test_creates_external_manifest(self): available_package = DummyFetchedPackage("foo", "1.1") sources_dict = self.sourcesDictForPackages([available_package]) === modified file 'linaro_image_tools/hwpack/tests/test_config.py' --- linaro_image_tools/hwpack/tests/test_config.py 2011-05-10 15:09:20 +0000 +++ linaro_image_tools/hwpack/tests/test_config.py 2011-06-17 12:28:43 +0000 @@ -30,6 +30,13 @@ valid_start = ( "[hwpack]\nname = ahwpack\npackages = foo\narchitectures = armel\n") + valid_start_v2 = valid_start + "format = 2.0\n" + valid_complete_v2 = (valid_start_v2 + + "u-boot-package = u-boot-linaro-s5pv310\n" \ + "u-boot-file = usr/lib/u-boot/smdkv310/" \ + "u-boot.bin\nserial_tty=ttySAC1\n" \ + "partition_layout = bootfs_rootfs\n") + valid_end = "[ubuntu]\nsources-entry = foo bar\n" def test_create(self): config = Config(StringIO()) @@ -170,6 +177,192 @@ "[ubuntu]\nsources-entry = foo bar\n") self.assertEqual(None, config.validate()) + def test_validate_supported_format(self): + config = self.get_config( + self.valid_start + + "\nformat = 0.9\n") + self.assertValidationError( + "Format version '0.9' is not supported.", config) + + def test_validate_invalid_u_boot_package_name(self): + config = self.get_config( + self.valid_start_v2 + "u-boot-package = ~~\n") + self.assertValidationError( + "Invalid value in u-boot-package in the [hwpack] section: ~~", + config) + + def test_validate_empty_u_boot_package(self): + config = self.get_config( + self.valid_start_v2 + "u-boot-package = \n") + self.assertValidationError( + "No u-boot-package in the [hwpack] section", config) + + def test_validate_no_u_boot_file(self): + config = self.get_config(self.valid_start_v2 + + "u-boot-package = u-boot-linaro-s5pv310\n") + self.assertValidationError("No u_boot_file in the [hwpack] section", + config) + + def test_validate_empty_u_boot_file(self): + config = self.get_config(self.valid_start_v2 + + "u-boot-package = u-boot-linaro-s5pv310\n" \ + "u-boot-file = \n") + self.assertValidationError("No u_boot_file in the [hwpack] section", config) + + def test_validate_invalid_u_boot_file(self): + config = self.get_config(self.valid_start_v2 + + "u-boot-package = u-boot-linaro-s5pv310\n" \ + "u-boot-file = ~~\n") + self.assertValidationError("Invalid path: ~~", config) + + def test_validate_partition_layout(self): + config = self.get_config(self.valid_start_v2 + + "u-boot-package = u-boot-linaro-s5pv310\n" \ + "u-boot-file = u-boot.bin\n" \ + "partition_layout = apafs_bananfs\n") + self.assertValidationError( + "Undefined partition layout apafs_bananfs in the [hwpack] " \ + "section. Valid partition layouts are bootfs_rootfs.", config) + + def test_validate_wired_interfaces(self): + self.assertTrue("XXX What is an invalid interface name?") + + def test_validate_wireless_interfaces(self): + self.assertTrue("XXX What is an invalid interface name?") + + def test_validate_serial_tty(self): + config = self.get_config(self.valid_start_v2 + + "u-boot-package = u-boot-linaro-s5pv310\n" \ + "u-boot-file = u-boot.bin\nserial_tty=tty\n") + self.assertValidationError("Invalid serial tty: tty", config) + config = self.get_config(self.valid_start_v2 + + "u-boot-package = u-boot-linaro-s5pv310\n" \ + "u-boot-file = u-boot.bin\n" \ + "serial_tty=ttxSAC1\n") + self.assertValidationError("Invalid serial tty: ttxSAC1", config) + + def test_validate_mmc_id(self): + config = self.get_config(self.valid_complete_v2 + + "mmc_id = x\n") + self.assertValidationError("Invalid mmc id x", config) + + def test_validate_kernel_addr(self): + config = self.get_config(self.valid_complete_v2 + + "kernel_addr = 0x8000000\n") + self.assertValidationError("Invalid kernel address: 0x8000000", config) + config = self.get_config(self.valid_complete_v2 + + "kernel_addr = 0x8000000x\n") + self.assertValidationError("Invalid kernel address: 0x8000000x", config) + config = self.get_config(self.valid_complete_v2 + + "kernel_addr = 80000000\n") + self.assertValidationError("Invalid kernel address: 80000000", config) + + def test_validate_initrd_addr(self): + config = self.get_config(self.valid_complete_v2 + + "initrd_addr = 0x8000000\n") + self.assertValidationError("Invalid initrd address: 0x8000000", config) + config = self.get_config(self.valid_complete_v2 + + "initrd_addr = 0x8000000x\n") + self.assertValidationError("Invalid initrd address: 0x8000000x", config) + config = self.get_config(self.valid_complete_v2 + + "initrd_addr = 80000000\n") + self.assertValidationError("Invalid initrd address: 80000000", config) + + def test_validate_load_addr(self): + config = self.get_config(self.valid_complete_v2 + + "load_addr = 0x8000000\n") + self.assertValidationError("Invalid load address: 0x8000000", config) + config = self.get_config(self.valid_complete_v2 + + "load_addr = 0x8000000x\n") + self.assertValidationError("Invalid load address: 0x8000000x", config) + config = self.get_config(self.valid_complete_v2 + + "load_addr = 80000000\n") + self.assertValidationError("Invalid load address: 80000000", config) + + def test_wired_interfaces(self): + config = self.get_config(self.valid_complete_v2 + + "wired_interfaces = eth0\n" + + self.valid_end) + config.validate() + self.assertEqual(["eth0"], config.wired_interfaces) + config = self.get_config(self.valid_complete_v2 + + "wired_interfaces = eth0 eth1 usb2\n" + + self.valid_end) + config.validate() + self.assertEqual(["eth0", "eth1", "usb2"], config.wired_interfaces) + + def test_wireless_interfaces(self): + config = self.get_config(self.valid_complete_v2 + + "wireless_interfaces = wlan0\n" + + self.valid_end) + config.validate() + self.assertEqual(["wlan0"], config.wireless_interfaces) + config = self.get_config(self.valid_complete_v2 + + "wireless_interfaces = wlan0 wl1 usb2\n" + + self.valid_end) + config.validate() + self.assertEqual(["wlan0", "wl1", "usb2"], config.wireless_interfaces) + + def test_partition_layout(self): + config = self.get_config(self.valid_complete_v2 + self.valid_end) + config.validate() + self.assertEqual("bootfs_rootfs", + config.partition_layout) + + def test_u_boot_file(self): + config = self.get_config(self.valid_complete_v2 + self.valid_end) + config.validate() + self.assertEqual("usr/lib/u-boot/smdkv310/u-boot.bin", + config.u_boot_file) + + def test_serial_tty(self): + config = self.get_config(self.valid_complete_v2 + self.valid_end) + config.validate() + self.assertEqual("ttySAC1", config.serial_tty) + + def test_mmc_id(self): + config = self.get_config(self.valid_complete_v2 + + "mmc_id = 1\n" + + self.valid_end) + config.validate() + self.assertEqual("1", config.mmc_id) + + def test_kernel_addr(self): + config = self.get_config(self.valid_complete_v2 + + "kernel_addr = 0x80000000\n" + + self.valid_end) + config.validate() + self.assertEqual("0x80000000", config.kernel_addr) + config = self.get_config(self.valid_complete_v2 + + "kernel_addr = 0x8aBcdEFf\n" + + self.valid_end) + config.validate() + self.assertEqual("0x8aBcdEFf", config.kernel_addr) + + def test_initrd_addr(self): + config = self.get_config(self.valid_complete_v2 + + "initrd_addr = 0x80000000\n" + + self.valid_end) + config.validate() + self.assertEqual("0x80000000", config.initrd_addr) + config = self.get_config(self.valid_complete_v2 + + "initrd_addr = 0x8aBcdEFf\n" + + self.valid_end) + config.validate() + self.assertEqual("0x8aBcdEFf", config.initrd_addr) + + def test_load_addr(self): + config = self.get_config(self.valid_complete_v2 + + "load_addr = 0x80000000\n" + + self.valid_end) + config.validate() + self.assertEqual("0x80000000", config.load_addr) + config = self.get_config(self.valid_complete_v2 + + "load_addr = 0x8aBcdEFf\n" + + self.valid_end) + config.validate() + self.assertEqual("0x8aBcdEFf", config.load_addr) + def test_name(self): config = self.get_config( "[hwpack]\nname = ahwpack\npackages = foo\n" === modified file 'linaro_image_tools/hwpack/tests/test_hardwarepack.py' --- linaro_image_tools/hwpack/tests/test_hardwarepack.py 2011-03-23 22:25:10 +0000 +++ linaro_image_tools/hwpack/tests/test_hardwarepack.py 2011-06-17 15:10:49 +0000 @@ -37,29 +37,32 @@ MatchesStructure, Not, ) +from linaro_image_tools.hwpack.hardwarepack_format import ( + HardwarePackFormatV1, + HardwarePackFormatV2, + ) class MetadataTests(TestCase): + def setUp(self): + super(MetadataTests, self).setUp() + self.metadata = Metadata("ahwpack", "3", "armel") def test_name(self): - metadata = Metadata("ahwpack", "3", "armel") - self.assertEqual("ahwpack", metadata.name) + self.assertEqual("ahwpack", self.metadata.name) def test_version(self): - metadata = Metadata("ahwpack", "3", "armel") - self.assertEqual("3", metadata.version) + self.assertEqual("3", self.metadata.version) def test_version_with_whitespace(self): self.assertRaises( AssertionError, Metadata, "ahwpack", "3 (with extras)", "armel") def test_architecture(self): - metadata = Metadata("ahwpack", "3", "armel") - self.assertEqual("armel", metadata.architecture) + self.assertEqual("armel", self.metadata.architecture) def test_default_origin_is_None(self): - metadata = Metadata("ahwpack", "4", "armel") - self.assertEqual(None, metadata.origin) + self.assertEqual(None, self.metadata.origin) def test_origin(self): metadata = Metadata("ahwpack", "4", "armel", origin="linaro") @@ -108,12 +111,85 @@ "SUPPORT=unsupported\n", str(metadata)) + def test_str_with_serial_tty(self): + metadata = Metadata("ahwpack", "4", "armel", + format=HardwarePackFormatV2()) + metadata.add_v2_config(serial_tty='ttyO2') + self.assertEqual( + "NAME=ahwpack\nVERSION=4\nARCHITECTURE=armel\n" + "SERIAL_TTY=ttyO2\n", + str(metadata)) + + def test_str_with_kernel_addr(self): + metadata = Metadata("ahwpack", "4", "armel", + format=HardwarePackFormatV2()) + metadata.add_v2_config(kernel_addr='0x80000000') + self.assertEqual( + "NAME=ahwpack\nVERSION=4\nARCHITECTURE=armel\n" + "KERNEL_ADDR=0x80000000\n", + str(metadata)) + + def test_str_with_initrd_addr(self): + metadata = Metadata("ahwpack", "4", "armel", + format=HardwarePackFormatV2()) + metadata.add_v2_config(initrd_addr='0x80000000') + self.assertEqual( + "NAME=ahwpack\nVERSION=4\nARCHITECTURE=armel\n" + "INITRD_ADDR=0x80000000\n", + str(metadata)) + + def test_str_with_load_addr(self): + metadata = Metadata("ahwpack", "4", "armel", + format=HardwarePackFormatV2()) + metadata.add_v2_config(load_addr='0x80000000') + self.assertEqual( + "NAME=ahwpack\nVERSION=4\nARCHITECTURE=armel\n" + "LOAD_ADDR=0x80000000\n", + str(metadata)) + + def test_str_with_wired_interfaces(self): + metadata = Metadata("ahwpack", "4", "armel", + format=HardwarePackFormatV2()) + metadata.add_v2_config(wired_interfaces=['eth0', 'usb0']) + self.assertEqual( + "NAME=ahwpack\nVERSION=4\nARCHITECTURE=armel\n" + "WIRED_INTERFACES=eth0 usb0\n", + str(metadata)) + + def test_str_with_wireless_interfaces(self): + metadata = Metadata("ahwpack", "4", "armel", + format=HardwarePackFormatV2()) + metadata.add_v2_config(wireless_interfaces=['wlan0', 'wl0']) + self.assertEqual( + "NAME=ahwpack\nVERSION=4\nARCHITECTURE=armel\n" + "WIRELESS_INTERFACES=wlan0 wl0\n", + str(metadata)) + + def test_str_with_partition_layout(self): + metadata = Metadata("ahwpack", "4", "armel", + format=HardwarePackFormatV2()) + metadata.add_v2_config(partition_layout='bootfs_rootfs') + self.assertEqual( + "NAME=ahwpack\nVERSION=4\nARCHITECTURE=armel\n" + "PARTITION_LAYOUT=bootfs_rootfs\n", + str(metadata)) + + def test_str_with_mmc_id(self): + metadata = Metadata("ahwpack", "4", "armel", + format=HardwarePackFormatV2()) + metadata.add_v2_config(mmc_id='1') + self.assertEqual( + "NAME=ahwpack\nVERSION=4\nARCHITECTURE=armel\n" + "MMC_ID=1\n", + str(metadata)) + def test_from_config(self): class Config: name = "foo" origin = "linaro" maintainer = "someone" support = "supported" + format = HardwarePackFormatV1() config = Config() metadata = Metadata.from_config(config, "2.0", "i386") self.assertEqual(config.name, metadata.name) @@ -130,14 +206,15 @@ super(HardwarePackTests, self).setUp() self.metadata = Metadata("ahwpack", "4", "armel") - def test_format_is_1_0(self): + def test_format_is_correct(self): + format = '1.0' hwpack = HardwarePack(self.metadata) - self.assertEqual("1.0", hwpack.FORMAT) + self.assertEqual(format, hwpack.format.__str__()) def test_format_has_no_spaces(self): hwpack = HardwarePack(self.metadata) - self.assertIs(None, re.search('\s', hwpack.FORMAT), - "hwpack.FORMAT contains spaces.") + self.assertIs(None, re.search('\s', hwpack.format.__str__()), + "hwpack.format contains spaces.") def test_filename(self): hwpack = HardwarePack(self.metadata) @@ -167,7 +244,7 @@ tf = self.get_tarfile(hwpack) self.assertThat( tf, - HardwarePackHasFile("FORMAT", content=hwpack.FORMAT+"\n")) + HardwarePackHasFile("FORMAT", content=hwpack.format.__str__()+"\n")) def test_creates_metadata_file(self): metadata = Metadata( === modified file 'linaro_image_tools/tests/fixtures.py' --- linaro_image_tools/tests/fixtures.py 2011-04-04 11:29:01 +0000 +++ linaro_image_tools/tests/fixtures.py 2011-06-20 09:10:10 +0000 @@ -20,6 +20,7 @@ import os import shutil import tempfile +from StringIO import StringIO from linaro_image_tools import cmd_runner @@ -68,8 +69,11 @@ # used in tests to make sure all callsites wait for their child. child_finished = True + def __init__(self, assert_child_finished=True): + self.assert_child_finished = assert_child_finished + def __call__(self, cmd, *args, **kwargs): - if not self.child_finished: + if self.assert_child_finished and not self.child_finished: raise AssertionError( "You should call wait() or communicate() to ensure " "the subprocess is finished before proceeding.") @@ -97,6 +101,9 @@ def commands_executed(self): return [' '.join(args) for args in self.calls] + @property + def stdin(self): + return StringIO() class MockCmdRunnerPopenFixture(MockSomethingFixture): """A test fixture which mocks cmd_runner.do_run with the given mock. @@ -104,13 +111,13 @@ If no mock is given, a MockCmdRunnerPopen instance is used. """ - def __init__(self): + def __init__(self, assert_child_finished=True): super(MockCmdRunnerPopenFixture, self).__init__( - cmd_runner, 'Popen', MockCmdRunnerPopen()) + cmd_runner, 'Popen', MockCmdRunnerPopen(assert_child_finished)) def tearDown(self): super(MockCmdRunnerPopenFixture, self).tearDown() - if not self.mock.child_finished: + if self.mock.assert_child_finished and not self.mock.child_finished: raise AssertionError( "You should call wait() or communicate() to ensure " "the subprocess is finished before proceeding.")