[Branch,~linaro-maintainers/linaro-image-tools/trunk] Rev 312: Merge lp:~jeremychang/linaro-image-tools/android adding experimental linaro

Message ID 20110405092813.19179.22398.launchpad@loganberry.canonical.com
State Accepted
Headers show

Commit Message

Alexander Sack April 5, 2011, 9:28 a.m.
Merge authors:
  Jeremy Chang (jeremychang)
Related merge proposals:
  https://code.launchpad.net/~jeremychang/linaro-image-tools/android/+merge/56113
  proposed by: Alexander Sack (asac)
  review: Approve - James Westby (james-w)
------------------------------------------------------------
revno: 312 [merge]
committer: Alexander Sack <asac@linaro.org>
branch nick: linaro-image-tools
timestamp: Tue 2011-04-05 11:26:47 +0200
message:
  Merge lp:~jeremychang/linaro-image-tools/android adding experimental linaro
  android platform image support; LP: #724207
added:
  linaro-android-media-create
modified:
  linaro_image_tools/media_create/__init__.py
  linaro_image_tools/media_create/boards.py
  linaro_image_tools/media_create/partitions.py
  linaro_image_tools/media_create/rootfs.py
  linaro_image_tools/media_create/unpack_binary_tarball.py


--
lp:linaro-image-tools
https://code.launchpad.net/~linaro-maintainers/linaro-image-tools/trunk

You are subscribed to branch lp:linaro-image-tools.
To unsubscribe from this branch go to https://code.launchpad.net/~linaro-maintainers/linaro-image-tools/trunk/+edit-subscription

Patch

=== added file 'linaro-android-media-create'
--- linaro-android-media-create	1970-01-01 00:00:00 +0000
+++ linaro-android-media-create	2011-03-31 19:03:31 +0000
@@ -0,0 +1,158 @@ 
+#!/usr/bin/env python
+# Copyright (C) 2011 Linaro
+#
+# Author: Jeremy Chang <jeremy.chang@linaro.org>
+#
+# This file is part of Linaro Image Tools.
+#
+# Linaro Image Tools is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Linaro Image Tools is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Linaro Image Tools.  If not, see <http://www.gnu.org/licenses/>.
+
+import atexit
+import os
+import sys
+import tempfile
+
+from linaro_image_tools import cmd_runner
+
+from linaro_image_tools.media_create.boards import android_board_configs
+from linaro_image_tools.media_create.check_device import (
+    confirm_device_selection_and_ensure_it_is_ready)
+from linaro_image_tools.media_create.hwpack import install_hwpacks
+from linaro_image_tools.media_create.partitions import (
+    Media,
+    setup_partitions,
+    setup_android_partitions,
+    get_uuid,
+    )
+from linaro_image_tools.media_create.populate_boot import populate_boot
+from linaro_image_tools.media_create.rootfs import populate_partition
+from linaro_image_tools.media_create.unpack_binary_tarball import (
+    unpack_android_binary_tarball
+    )
+from linaro_image_tools.media_create import get_android_args_parser
+from linaro_image_tools.utils import ensure_command, is_arm_host
+
+
+
+# Just define the global variables
+TMP_DIR = None
+ROOTFS_DIR = None
+BOOT_DISK = None
+ROOT_DISK = None
+SYSTEM_DISK = None
+CACHE_DISK = None
+DATA_DISK = None
+SDCARD_DISK = None
+
+
+# Registered as the first atexit handler as we want this to be the last
+# handler to execute.
+@atexit.register
+def cleanup_tempdir():
+    """Remove TEMP_DIR with all its contents.
+
+    Before doing so, make sure BOOT_DISK and ROOT_DISK are not mounted.
+    """
+    devnull = open('/dev/null', 'w')
+    # ignore non-zero return codes
+    for disk in BOOT_DISK, ROOT_DISK, SYSTEM_DISK, CACHE_DISK, DATA_DISK, \
+                SDCARD_DISK:
+        if disk is not None:
+            try:
+                cmd_runner.run(['umount', disk],
+                      stdout=devnull, stderr=devnull, as_root=True).wait()
+            except cmd_runner.SubcommandNonZeroReturnValue:
+                pass
+    # Remove TMP_DIR as root because some files written there are
+    # owned by root.
+    if TMP_DIR is not None:
+        cmd_runner.run(['rm', '-rf', TMP_DIR], as_root=True).wait()
+
+
+def ensure_required_commands(args):
+    """Ensure we have the commands that we know are going to be used."""
+    required_commands = [
+        'mkfs.vfat', 'sfdisk', 'mkimage', 'parted']
+    if not is_arm_host():
+        required_commands.append('qemu-arm-static')
+        required_commands.append('qemu-img')
+    if args.rootfs in ['ext2', 'ext3', 'ext4']:
+        required_commands.append('mkfs.%s' % args.rootfs)
+    else:
+        required_commands.append('mkfs.btrfs')
+    for command in required_commands:
+        ensure_command(command)
+
+
+if __name__ == '__main__':
+    parser = get_android_args_parser()
+    print "===================================================================="
+    print "            linaro-android-media-create is EXPERIMENTAL             "
+    print "                                                                    "
+    print " The command line parameters as well as the distribution format will" 
+    print " be changed and we will not keep backward compatibility for the     " 
+    print " current version here.                                              "
+    print "===================================================================="
+    args = parser.parse_args()
+
+    # If --help was specified this won't execute.
+    # Create temp dir and initialize rest of path vars.
+    TMP_DIR = tempfile.mkdtemp()
+    ROOT_DIR = os.path.join(TMP_DIR, 'root')
+    SYSTEM_DIR = os.path.join(TMP_DIR, 'system')
+    DATA_DIR = os.path.join(TMP_DIR, 'data')
+
+    BOOT_DISK = os.path.join(TMP_DIR, 'boot-disc')
+    ROOT_DISK = os.path.join(TMP_DIR, 'root-disc')
+    SYSTEM_DISK = os.path.join(TMP_DIR, 'system-disc')
+    CACHE_DISK = os.path.join(TMP_DIR, 'cache-disc')
+    DATA_DISK = os.path.join(TMP_DIR, 'userdata-disc')
+    SDCARD_DISK = os.path.join(TMP_DIR, 'sdcard-disc')
+
+    board_config = android_board_configs[args.board]
+
+    ensure_required_commands(args)
+
+    media = Media(args.device)
+    if media.is_block_device:
+        if not confirm_device_selection_and_ensure_it_is_ready(args.device):
+            sys.exit(1)
+    elif not args.should_format_rootfs or not args.should_format_bootfs:
+        print ("Do not use --no-boot or --no-part in conjunction with "
+               "--image_file.")
+        sys.exit(1)
+    else:
+        # All good, move on.
+        pass
+
+
+    cmd_runner.run(['mkdir', '-p', ROOT_DIR]).wait()
+    cmd_runner.run(['mkdir', '-p', SYSTEM_DIR]).wait()
+    cmd_runner.run(['mkdir', '-p', DATA_DIR]).wait()
+
+    unpack_android_binary_tarball(args.root, ROOT_DIR)
+    unpack_android_binary_tarball(args.system, SYSTEM_DIR)
+    unpack_android_binary_tarball(args.userdata, DATA_DIR)
+
+    # Create partitions    
+    boot_partition, root_partition, system_partition, cache_partition, \
+        data_partition, sdcard_partition = setup_android_partitions( \
+        board_config, media, args.boot_label, args.rfs_label,
+        args.rootfs, args.should_create_partitions, args.should_format_bootfs,
+        args.should_format_rootfs, args.should_align_boot_part)
+
+    populate_partition(ROOT_DIR, ROOT_DISK, root_partition)
+    populate_partition(SYSTEM_DIR + "/system", SYSTEM_DISK, system_partition)
+    populate_partition(DATA_DIR + "/data", DATA_DISK, data_partition)
+    print "Done creating Linaro Android image on %s" % args.device

=== modified file 'linaro_image_tools/media_create/__init__.py'
--- linaro_image_tools/media_create/__init__.py	2011-03-23 22:25:10 +0000
+++ linaro_image_tools/media_create/__init__.py	2011-03-31 19:03:31 +0000
@@ -20,9 +20,11 @@ 
 import argparse
 
 from linaro_image_tools.media_create.boards import board_configs
+from linaro_image_tools.media_create.boards import android_board_configs
 
 
 KNOWN_BOARDS = board_configs.keys()
+ANDROID_KNOWN_BOARDS = android_board_configs.keys()
 
 
 class Live256MegsAction(argparse.Action):
@@ -110,3 +112,46 @@ 
         action='store_true',
         help='Align boot partition too (might break older x-loaders).')
     return parser
+
+def get_android_args_parser():
+    """Get the ArgumentParser for the arguments given on the command line."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--mmc', required=True, dest='device', help='The storage device to use.')
+    parser.add_argument(
+        '--dev', required=True, dest='board', choices=ANDROID_KNOWN_BOARDS,
+        help='Generate an SD card or image for the given board.')
+    parser.add_argument(
+        '--rootfs', default='ext4', choices=['ext3', 'ext4'],
+        help='Type of filesystem to use for the rootfs')
+    parser.add_argument(
+        '--rfs_label', default='rootfs',
+        help='Label to use for the root filesystem.')
+    parser.add_argument(
+        '--boot_label', default='boot',
+        help='Label to use for the boot filesystem.')
+
+    parser.add_argument(
+        '--system', default='system.tar.bz2', required=True,
+        help=('The tarball containing the Android system paritition'))
+    parser.add_argument(
+        '--userdata', default='userdata.tar.bz2', required=True,
+        help=('The tarball containing the Android data paritition'))
+    parser.add_argument(
+        '--root', default='root.tar.bz2', required=True,
+        help=('The tarball containing the Android root partition'))
+
+    parser.add_argument(
+        '--no-rootfs', dest='should_format_rootfs', action='store_false',
+        help='Do not deploy the root filesystem.')
+    parser.add_argument(
+        '--no-bootfs', dest='should_format_bootfs', action='store_false',
+        help='Do not deploy the boot filesystem.')
+    parser.add_argument(
+        '--no-part', dest='should_create_partitions', action='store_false',
+        help='Reuse existing partitions on the given media.')
+    parser.add_argument(
+        '--align-boot-part', dest='should_align_boot_part',
+        action='store_true',
+        help='Align boot partition too (might break older x-loaders).')
+    return parser

=== modified file 'linaro_image_tools/media_create/boards.py'
--- linaro_image_tools/media_create/boards.py	2011-04-04 16:22:10 +0000
+++ linaro_image_tools/media_create/boards.py	2011-04-05 09:26:47 +0000
@@ -168,6 +168,9 @@ 
         else:
             partition_type = '0x0E'
 
+        BOOT_MIN_SIZE_S = align_up(50 * 1024 * 1024, SECTOR_SIZE) / SECTOR_SIZE
+        ROOT_MIN_SIZE_S = align_up(50 * 1024 * 1024, SECTOR_SIZE) / SECTOR_SIZE
+            
         # align on sector 63 for compatibility with broken versions of x-loader
         # unless align_boot_part is set
         boot_align = 63
@@ -182,6 +185,7 @@ 
         # there should still be enough room
         boot_len = boot_len - boot_len % 2
         boot_end = boot_start + boot_len - 1
+
         # we ignore _root_end / _root_len and return a sfdisk command to
         # instruct the use of all remaining space; XXX if we had some root size
         # config, we could do something more sensible
@@ -191,6 +195,51 @@ 
         return '%s,%s,%s,*\n%s,,,-' % (
             boot_start, boot_len, partition_type, root_start)
 
+    @classmethod
+    def get_android_sfdisk_cmd(cls, should_align_boot_part=False):
+        if cls.fat_size == 32:
+            partition_type = '0x0C'
+        else:
+            partition_type = '0x0E'
+
+        BOOT_MIN_SIZE_S = align_up(128 * 1024 * 1024, SECTOR_SIZE) / SECTOR_SIZE
+        ROOT_MIN_SIZE_S = align_up(128 * 1024 * 1024, SECTOR_SIZE) / SECTOR_SIZE
+        SYSTEM_MIN_SIZE_S = align_up(256 * 1024 * 1024, SECTOR_SIZE) / SECTOR_SIZE
+        CACHE_MIN_SIZE_S = align_up(256 * 1024 * 1024, SECTOR_SIZE) / SECTOR_SIZE
+        USERDATA_MIN_SIZE_S = align_up(512 * 1024 * 1024, SECTOR_SIZE) / SECTOR_SIZE
+        SDCARD_MIN_SIZE_S = align_up(512 * 1024 * 1024, SECTOR_SIZE) / SECTOR_SIZE
+
+        # align on sector 63 for compatibility with broken versions of x-loader
+        # unless align_boot_part is set
+        boot_align = 63
+        if should_align_boot_part:
+            boot_align = PART_ALIGN_S
+
+        # can only start on sector 1 (sector 0 is MBR / partition table)
+        boot_start, boot_end, boot_len = align_partition(
+            1, BOOT_MIN_SIZE_S, boot_align, PART_ALIGN_S)
+        # apparently OMAP3 ROMs require the vfat length to be an even number
+        # of sectors (multiple of 1 KiB); decrease the length if it's odd,
+        # there should still be enough room
+        boot_len = boot_len - boot_len % 2
+        boot_end = boot_start + boot_len - 1
+
+        root_start, _root_end, _root_len = align_partition(
+            boot_end + 1, ROOT_MIN_SIZE_S, PART_ALIGN_S, PART_ALIGN_S)
+        system_start, _system_end, _system_len = align_partition(
+            _root_end + 1, SYSTEM_MIN_SIZE_S, PART_ALIGN_S, PART_ALIGN_S)
+        cache_start, _cache_end, _cache_len = align_partition(
+            _system_end + 1, CACHE_MIN_SIZE_S, PART_ALIGN_S, PART_ALIGN_S)
+        userdata_start, _userdata_end, _userdata_len = align_partition(
+            _cache_end + 1, USERDATA_MIN_SIZE_S, PART_ALIGN_S, PART_ALIGN_S)
+        sdcard_start, _sdcard_end, _sdcard_len = align_partition(
+            _userdata_end + 1, SDCARD_MIN_SIZE_S, PART_ALIGN_S, PART_ALIGN_S)
+ 
+        return '%s,%s,%s,*\n%s,%s,L\n%s,%s,L\n%s,-,E\n%s,%s,L\n%s,%s,L\n%s,,,-' % (
+            boot_start, boot_len, partition_type, root_start, _root_len, 
+            system_start, _system_len, cache_start, cache_start, _cache_len,
+            userdata_start, _userdata_len, sdcard_start)
+
     @classproperty
     def bootcmd(cls):
         """Get the bootcmd for this board.
@@ -641,6 +690,11 @@ 
     'smdkv310': SMDKV310Config,
     }
 
+android_board_configs = {
+    'beagle': BeagleConfig,
+    'panda': PandaConfig,
+    }
+
 
 def _dd(input_file, output_file, block_size=SECTOR_SIZE, count=None, seek=None,
         skip=None):

=== modified file 'linaro_image_tools/media_create/partitions.py'
--- linaro_image_tools/media_create/partitions.py	2011-03-24 10:10:38 +0000
+++ linaro_image_tools/media_create/partitions.py	2011-03-31 18:44:16 +0000
@@ -41,6 +41,58 @@ 
 UDISKS = "org.freedesktop.UDisks"
 
 
+def setup_android_partitions(board_config, media, bootfs_label,
+                     rootfs_label, rootfs_type, should_create_partitions,
+                     should_format_bootfs, should_format_rootfs,
+                     should_align_boot_part=False):
+    cylinders = None
+
+    if should_create_partitions:
+        create_partitions(
+            board_config, media, HEADS, SECTORS, cylinders,
+            should_align_boot_part=should_align_boot_part, image_type="ANDROID")
+
+    bootfs, rootfs, system, cache, data, sdcard = \
+        get_android_partitions_for_media (media, board_config)
+    ensure_partition_is_not_mounted(bootfs)
+    ensure_partition_is_not_mounted(rootfs)
+    ensure_partition_is_not_mounted(system)
+    ensure_partition_is_not_mounted(cache)
+    ensure_partition_is_not_mounted(data)
+    ensure_partition_is_not_mounted(sdcard)
+
+    if should_format_bootfs:
+        print "\nFormating boot partition\n"
+        proc = cmd_runner.run(
+            ['mkfs.vfat', '-F', str(board_config.fat_size), bootfs, '-n',
+             bootfs_label],
+            as_root=True)
+        proc.wait()
+
+    if should_format_rootfs:
+        print "\nFormating root partition\n"
+        mkfs = 'mkfs.%s' % rootfs_type
+        proc = cmd_runner.run(
+            [mkfs, rootfs, '-L', rootfs_label],
+            as_root=True)
+        proc.wait()
+
+    ext4_partitions = {"system": system, "cache": cache, "userdata": data}
+    for label, dev in ext4_partitions.iteritems():
+        mkfs = 'mkfs.%s' % "ext4"
+        proc = cmd_runner.run(
+            [mkfs, dev, '-L', label],
+            as_root=True)
+        proc.wait()
+
+    proc = cmd_runner.run(
+        ['mkfs.vfat', '-F', str(board_config.fat_size), sdcard, '-n',
+         "sdcard"],
+        as_root=True)
+    proc.wait()
+
+    return bootfs, rootfs, system, cache, data, sdcard
+
 # I wonder if it'd make sense to convert this into a small shim which calls
 # the appropriate function for the given type of device?  I think it's still
 # small enough that there's not much benefit in doing that, but if it grows we
@@ -212,6 +264,35 @@ 
         "Couldn't find root partition on %s" % image_file)
     return vfat_size, vfat_offset, linux_size, linux_offset
 
+def get_android_partitions_for_media(media, board_config):
+    """Return the device files for all the Android partitions of media.
+
+    For boot we use partition number 1 plus the board's defined partition
+    offset and for root we use partition number 2 plus the board's offset.
+
+    This function must only be used for block devices.
+    """
+    assert media.is_block_device, (
+        "This function must only be used for block devices")
+
+    boot_partition = _get_device_file_for_partition_number(
+        media.path, 1 + board_config.mmc_part_offset)
+    root_partition = _get_device_file_for_partition_number(
+        media.path, 2 + board_config.mmc_part_offset)
+    system_partition = _get_device_file_for_partition_number(
+        media.path, 3 + board_config.mmc_part_offset)
+    cache_partition = _get_device_file_for_partition_number(
+        media.path, 5 + board_config.mmc_part_offset)
+    data_partition = _get_device_file_for_partition_number(
+        media.path, 6 + board_config.mmc_part_offset)
+    sdcard_partition = _get_device_file_for_partition_number(
+        media.path, 7 + board_config.mmc_part_offset)
+
+    assert boot_partition is not None and root_partition is not None, (
+        "Could not find boot/root partition for %s" % media.path)
+
+    return boot_partition, root_partition, system_partition, \
+        cache_partition, data_partition, sdcard_partition
 
 def get_boot_and_root_partitions_for_media(media, board_config):
     """Return the device files for the boot and root partitions of media.
@@ -307,7 +388,7 @@ 
 
 
 def create_partitions(board_config, media, heads, sectors, cylinders=None,
-                      should_align_boot_part=False):
+                      should_align_boot_part=False, image_type=None):
     """Partition the given media according to the board requirements.
 
     :param board_config: A BoardConfig class.
@@ -326,8 +407,13 @@ 
             ['parted', '-s', media.path, 'mklabel', 'msdos'], as_root=True)
         proc.wait()
 
-    sfdisk_cmd = board_config.get_sfdisk_cmd(
-        should_align_boot_part=should_align_boot_part)
+    if image_type == "ANDROID":
+        sfdisk_cmd = board_config.get_android_sfdisk_cmd(
+            should_align_boot_part=should_align_boot_part)
+    else:
+        sfdisk_cmd = board_config.get_sfdisk_cmd(
+            should_align_boot_part=should_align_boot_part)
+
     run_sfdisk_commands(sfdisk_cmd, heads, sectors, cylinders, media.path)
 
     # Sync and sleep to wait for the partition to settle.

=== modified file 'linaro_image_tools/media_create/rootfs.py'
--- linaro_image_tools/media_create/rootfs.py	2011-04-04 14:00:26 +0000
+++ linaro_image_tools/media_create/rootfs.py	2011-04-05 09:26:47 +0000
@@ -23,6 +23,13 @@ 
 
 from linaro_image_tools import cmd_runner
 
+def populate_partition(content_dir, root_disk, partition):
+    os.makedirs(root_disk)
+    cmd_runner.run(['mount', partition, root_disk], as_root=True).wait()
+    move_contents(content_dir, root_disk)
+    cmd_runner.run(['sync']).wait()
+    cmd_runner.run(['umount', root_disk], as_root=True).wait()
+
 
 def rootfs_mount_options(rootfs_type):
     """Return mount options for the specific rootfs type."""

=== modified file 'linaro_image_tools/media_create/unpack_binary_tarball.py'
--- linaro_image_tools/media_create/unpack_binary_tarball.py	2011-03-24 10:10:38 +0000
+++ linaro_image_tools/media_create/unpack_binary_tarball.py	2011-03-08 15:32:43 +0000
@@ -20,6 +20,13 @@ 
 from linaro_image_tools import cmd_runner
 
 
+def unpack_android_binary_tarball(tarball, unpack_dir, as_root=True):
+    proc = cmd_runner.run(
+        ['tar', '--numeric-owner', '-C', unpack_dir, '-jxf', tarball],
+        as_root=as_root)
+    proc.wait()
+    return proc.returncode
+
 def unpack_binary_tarball(tarball, unpack_dir, as_root=True):
     proc = cmd_runner.run(
         ['tar', '--numeric-owner', '-C', unpack_dir, '-xf', tarball],