diff mbox series

[v2,04/18] tests/docker: update debian-arm64-cross with lci-tool

Message ID 20220225172021.3493923-5-alex.bennee@linaro.org
State Superseded
Headers show
Series testing and semihosting pre-PR | expand

Commit Message

Alex Bennée Feb. 25, 2022, 5:20 p.m. UTC
Using lci-tool update debian-arm64-cross to a Debian 11 based system.
As a result we can drop debian-arm64-test-cross just for building
tests.

Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
Reviewed-by: Richard Henderson <richard.henderson@linaro.org>
Message-Id: <20220211160309.335014-5-alex.bennee@linaro.org>
---
 .gitlab-ci.d/container-cross.yml              |  10 +-
 tests/docker/Makefile.include                 |   3 -
 .../dockerfiles/debian-arm64-cross.docker     | 186 +++++++++++++++---
 .../debian-arm64-test-cross.docker            |  13 --
 tests/lcitool/refresh                         |  11 ++
 tests/tcg/configure.sh                        |   2 +-
 6 files changed, 173 insertions(+), 52 deletions(-)
 delete mode 100644 tests/docker/dockerfiles/debian-arm64-test-cross.docker

Comments

Thomas Huth Feb. 28, 2022, 8:39 a.m. UTC | #1
On 25/02/2022 18.20, Alex Bennée wrote:
> Using lci-tool update debian-arm64-cross to a Debian 11 based system.
> As a result we can drop debian-arm64-test-cross just for building
> tests.

Good idea!

One comment below...

[...]
> diff --git a/tests/docker/dockerfiles/debian-arm64-cross.docker b/tests/docker/dockerfiles/debian-arm64-cross.docker
> index 166e24df13..589510a7be 100644
> --- a/tests/docker/dockerfiles/debian-arm64-cross.docker
> +++ b/tests/docker/dockerfiles/debian-arm64-cross.docker
> @@ -1,32 +1,166 @@
> +# THIS FILE WAS AUTO-GENERATED
>   #
> -# Docker arm64 cross-compiler target
> +#  $ lcitool dockerfile --layers all --cross aarch64 debian-11 qemu
>   #
> -# This docker target builds on the debian Buster base image.
> -#
> -FROM qemu/debian10
> +# https://gitlab.com/libvirt/libvirt-ci
>   
> -# Add the foreign architecture we want and install dependencies
> -RUN dpkg --add-architecture arm64
> -RUN apt update && \
> -    DEBIAN_FRONTEND=noninteractive eatmydata \
> -    apt install -y --no-install-recommends \
> -        crossbuild-essential-arm64
> -RUN apt update && \
> -    DEBIAN_FRONTEND=noninteractive eatmydata \
> -    apt build-dep -yy -a arm64 --arch-only qemu
> +FROM docker.io/library/debian:11-slim
>   
> -# Specify the cross prefix for this image (see tests/docker/common.rc)
> -ENV QEMU_CONFIGURE_OPTS --cross-prefix=aarch64-linux-gnu-
> -ENV DEF_TARGET_LIST aarch64-softmmu,aarch64-linux-user
> +RUN export DEBIAN_FRONTEND=noninteractive && \
> +    apt-get update && \
> +    apt-get install -y eatmydata && \
> +    eatmydata apt-get dist-upgrade -y && \
> +    eatmydata apt-get install --no-install-recommends -y \
> +            bash \
> +            bc \
> +            bsdextrautils \
> +            bzip2 \
> +            ca-certificates \
> +            ccache \
> +            dbus \
> +            debianutils \
> +            diffutils \
> +            exuberant-ctags \
> +            findutils \
> +            gcovr \
> +            genisoimage \
> +            gettext \
> +            git \
> +            hostname \
> +            libpcre2-dev \
> +            libspice-protocol-dev \
> +            libtest-harness-perl \
> +            llvm \
> +            locales \
> +            make \
> +            meson \
> +            ncat \
> +            ninja-build \
> +            openssh-client \
> +            perl-base \
> +            pkgconf \
> +            python3 \
> +            python3-numpy \
> +            python3-opencv \
> +            python3-pillow \
> +            python3-pip \
> +            python3-sphinx \
> +            python3-sphinx-rtd-theme \
> +            python3-venv \
> +            python3-yaml \
> +            rpm2cpio \
> +            sed \
> +            sparse \
> +            tar \
> +            tesseract-ocr \
> +            tesseract-ocr-eng \
> +            texinfo && \
> +    eatmydata apt-get autoremove -y && \
> +    eatmydata apt-get autoclean -y && \
> +    sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \
> +    dpkg-reconfigure locales
>   
> -RUN apt update && \
> -    DEBIAN_FRONTEND=noninteractive eatmydata \
> -    apt install -y --no-install-recommends \
> -        libbz2-dev:arm64 \
> -        liblzo2-dev:arm64 \
> -        librdmacm-dev:arm64 \
> -        libsnappy-dev:arm64 \
> -        libxen-dev:arm64
> +ENV LANG "en_US.UTF-8"
> +ENV MAKE "/usr/bin/make"
> +ENV NINJA "/usr/bin/ninja"
> +ENV PYTHON "/usr/bin/python3"
> +ENV CCACHE_WRAPPERSDIR "/usr/libexec/ccache-wrappers"
>   
> -# nettle
> -ENV QEMU_CONFIGURE_OPTS $QEMU_CONFIGURE_OPTS --enable-nettle
> +RUN export DEBIAN_FRONTEND=noninteractive && \
> +    dpkg --add-architecture arm64 && \
> +    eatmydata apt-get update && \
> +    eatmydata apt-get dist-upgrade -y && \
> +    eatmydata apt-get install --no-install-recommends -y dpkg-dev && \
> +    eatmydata apt-get install --no-install-recommends -y \
> +            g++-aarch64-linux-gnu \
> +            gcc-aarch64-linux-gnu \
> +            libaio-dev:arm64 \
> +            libasan5:arm64 \
> +            libasound2-dev:arm64 \
> +            libattr1-dev:arm64 \
> +            libbpf-dev:arm64 \
> +            libbrlapi-dev:arm64 \
> +            libbz2-dev:arm64 \
> +            libc6-dev:arm64 \
> +            libcacard-dev:arm64 \
> +            libcap-ng-dev:arm64 \
> +            libcapstone-dev:arm64 \
> +            libcurl4-gnutls-dev:arm64 \
> +            libdaxctl-dev:arm64 \
> +            libdrm-dev:arm64 \
> +            libepoxy-dev:arm64 \
> +            libfdt-dev:arm64 \
> +            libffi-dev:arm64 \
> +            libfuse3-dev:arm64 \
> +            libgbm-dev:arm64 \
> +            libgcrypt20-dev:arm64 \
> +            libglib2.0-dev:arm64 \
> +            libglusterfs-dev:arm64 \
> +            libgnutls28-dev:arm64 \
> +            libgtk-3-dev:arm64 \
> +            libibumad-dev:arm64 \
> +            libibverbs-dev:arm64 \
> +            libiscsi-dev:arm64 \
> +            libjemalloc-dev:arm64 \
> +            libjpeg62-turbo-dev:arm64 \
> +            liblttng-ust-dev:arm64 \
> +            liblzo2-dev:arm64 \
> +            libncursesw5-dev:arm64 \
> +            libnfs-dev:arm64 \
> +            libnuma-dev:arm64 \
> +            libpam0g-dev:arm64 \
> +            libpixman-1-dev:arm64 \
> +            libpng-dev:arm64 \
> +            libpulse-dev:arm64 \
> +            librbd-dev:arm64 \
> +            librdmacm-dev:arm64 \
> +            libsasl2-dev:arm64 \
> +            libsdl2-dev:arm64 \
> +            libsdl2-image-dev:arm64 \
> +            libseccomp-dev:arm64 \
> +            libselinux1-dev:arm64 \
> +            libslirp-dev:arm64 \
> +            libsnappy-dev:arm64 \
> +            libspice-server-dev:arm64 \
> +            libssh-gcrypt-dev:arm64 \
> +            libsystemd-dev:arm64 \
> +            libtasn1-6-dev:arm64 \
> +            libubsan1:arm64 \
> +            libudev-dev:arm64 \
> +            liburing-dev:arm64 \
> +            libusb-1.0-0-dev:arm64 \
> +            libusbredirhost-dev:arm64 \
> +            libvdeplug-dev:arm64 \
> +            libvirglrenderer-dev:arm64 \
> +            libvte-2.91-dev:arm64 \
> +            libxen-dev:arm64 \
> +            libzstd-dev:arm64 \
> +            nettle-dev:arm64 \
> +            systemtap-sdt-dev:arm64 \
> +            xfslibs-dev:arm64 \
> +            zlib1g-dev:arm64 && \
> +    eatmydata apt-get autoremove -y && \
> +    eatmydata apt-get autoclean -y && \
> +    mkdir -p /usr/local/share/meson/cross && \
> +    echo "[binaries]\n\
> +c = '/usr/bin/aarch64-linux-gnu-gcc'\n\

Is it really just "c =" or should this be "cc =" ?

> +ar = '/usr/bin/aarch64-linux-gnu-gcc-ar'\n\
> +strip = '/usr/bin/aarch64-linux-gnu-strip'\n\
> +pkgconfig = '/usr/bin/aarch64-linux-gnu-pkg-config'\n\
> +\n\
> +[host_machine]\n\
> +system = 'linux'\n\
> +cpu_family = 'aarch64'\n\
> +cpu = 'aarch64'\n\
> +endian = 'little'" > /usr/local/share/meson/cross/aarch64-linux-gnu && \
> +    dpkg-query --showformat '${Package}_${Version}_${Architecture}\n' --show > /packages.txt && \
> +    mkdir -p /usr/libexec/ccache-wrappers && \
> +    ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/aarch64-linux-gnu-c++ && \
> +    ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/aarch64-linux-gnu-cc && \
> +    ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/aarch64-linux-gnu-g++ && \
> +    ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/aarch64-linux-gnu-gcc
> +
> +ENV ABI "aarch64-linux-gnu"
> +ENV MESON_OPTS "--cross-file=aarch64-linux-gnu"
> +ENV QEMU_CONFIGURE_OPTS --cross-prefix=aarch64-linux-gnu-
> +ENV DEF_TARGET_LIST aarch64-softmmu,aarch64-linux-user

  Thomas
Daniel P. Berrangé Feb. 28, 2022, 9:20 a.m. UTC | #2
On Mon, Feb 28, 2022 at 09:39:29AM +0100, Thomas Huth wrote:
> On 25/02/2022 18.20, Alex Bennée wrote:
> > Using lci-tool update debian-arm64-cross to a Debian 11 based system.
> > As a result we can drop debian-arm64-test-cross just for building
> > tests.
> 
> Good idea!
> 
> One comment below...
> 
> [...]
> > diff --git a/tests/docker/dockerfiles/debian-arm64-cross.docker b/tests/docker/dockerfiles/debian-arm64-cross.docker
> > index 166e24df13..589510a7be 100644
> > --- a/tests/docker/dockerfiles/debian-arm64-cross.docker
> > +++ b/tests/docker/dockerfiles/debian-arm64-cross.docker
> > @@ -1,32 +1,166 @@
> > +# THIS FILE WAS AUTO-GENERATED
> >   #
> > -# Docker arm64 cross-compiler target
> > +#  $ lcitool dockerfile --layers all --cross aarch64 debian-11 qemu
> >   #
> > -# This docker target builds on the debian Buster base image.
> > -#
> > -FROM qemu/debian10
> > +# https://gitlab.com/libvirt/libvirt-ci
> > -# Add the foreign architecture we want and install dependencies
> > -RUN dpkg --add-architecture arm64
> > -RUN apt update && \
> > -    DEBIAN_FRONTEND=noninteractive eatmydata \
> > -    apt install -y --no-install-recommends \
> > -        crossbuild-essential-arm64
> > -RUN apt update && \
> > -    DEBIAN_FRONTEND=noninteractive eatmydata \
> > -    apt build-dep -yy -a arm64 --arch-only qemu
> > +FROM docker.io/library/debian:11-slim
> > -# Specify the cross prefix for this image (see tests/docker/common.rc)
> > -ENV QEMU_CONFIGURE_OPTS --cross-prefix=aarch64-linux-gnu-
> > -ENV DEF_TARGET_LIST aarch64-softmmu,aarch64-linux-user
> > +RUN export DEBIAN_FRONTEND=noninteractive && \
> > +    apt-get update && \
> > +    apt-get install -y eatmydata && \
> > +    eatmydata apt-get dist-upgrade -y && \
> > +    eatmydata apt-get install --no-install-recommends -y \
> > +            bash \
> > +            bc \
> > +            bsdextrautils \
> > +            bzip2 \
> > +            ca-certificates \
> > +            ccache \
> > +            dbus \
> > +            debianutils \
> > +            diffutils \
> > +            exuberant-ctags \
> > +            findutils \
> > +            gcovr \
> > +            genisoimage \
> > +            gettext \
> > +            git \
> > +            hostname \
> > +            libpcre2-dev \
> > +            libspice-protocol-dev \
> > +            libtest-harness-perl \
> > +            llvm \
> > +            locales \
> > +            make \
> > +            meson \
> > +            ncat \
> > +            ninja-build \
> > +            openssh-client \
> > +            perl-base \
> > +            pkgconf \
> > +            python3 \
> > +            python3-numpy \
> > +            python3-opencv \
> > +            python3-pillow \
> > +            python3-pip \
> > +            python3-sphinx \
> > +            python3-sphinx-rtd-theme \
> > +            python3-venv \
> > +            python3-yaml \
> > +            rpm2cpio \
> > +            sed \
> > +            sparse \
> > +            tar \
> > +            tesseract-ocr \
> > +            tesseract-ocr-eng \
> > +            texinfo && \
> > +    eatmydata apt-get autoremove -y && \
> > +    eatmydata apt-get autoclean -y && \
> > +    sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \
> > +    dpkg-reconfigure locales
> > -RUN apt update && \
> > -    DEBIAN_FRONTEND=noninteractive eatmydata \
> > -    apt install -y --no-install-recommends \
> > -        libbz2-dev:arm64 \
> > -        liblzo2-dev:arm64 \
> > -        librdmacm-dev:arm64 \
> > -        libsnappy-dev:arm64 \
> > -        libxen-dev:arm64
> > +ENV LANG "en_US.UTF-8"
> > +ENV MAKE "/usr/bin/make"
> > +ENV NINJA "/usr/bin/ninja"
> > +ENV PYTHON "/usr/bin/python3"
> > +ENV CCACHE_WRAPPERSDIR "/usr/libexec/ccache-wrappers"
> > -# nettle
> > -ENV QEMU_CONFIGURE_OPTS $QEMU_CONFIGURE_OPTS --enable-nettle
> > +RUN export DEBIAN_FRONTEND=noninteractive && \
> > +    dpkg --add-architecture arm64 && \
> > +    eatmydata apt-get update && \
> > +    eatmydata apt-get dist-upgrade -y && \
> > +    eatmydata apt-get install --no-install-recommends -y dpkg-dev && \
> > +    eatmydata apt-get install --no-install-recommends -y \
> > +            g++-aarch64-linux-gnu \
> > +            gcc-aarch64-linux-gnu \
> > +            libaio-dev:arm64 \
> > +            libasan5:arm64 \
> > +            libasound2-dev:arm64 \
> > +            libattr1-dev:arm64 \
> > +            libbpf-dev:arm64 \
> > +            libbrlapi-dev:arm64 \
> > +            libbz2-dev:arm64 \
> > +            libc6-dev:arm64 \
> > +            libcacard-dev:arm64 \
> > +            libcap-ng-dev:arm64 \
> > +            libcapstone-dev:arm64 \
> > +            libcurl4-gnutls-dev:arm64 \
> > +            libdaxctl-dev:arm64 \
> > +            libdrm-dev:arm64 \
> > +            libepoxy-dev:arm64 \
> > +            libfdt-dev:arm64 \
> > +            libffi-dev:arm64 \
> > +            libfuse3-dev:arm64 \
> > +            libgbm-dev:arm64 \
> > +            libgcrypt20-dev:arm64 \
> > +            libglib2.0-dev:arm64 \
> > +            libglusterfs-dev:arm64 \
> > +            libgnutls28-dev:arm64 \
> > +            libgtk-3-dev:arm64 \
> > +            libibumad-dev:arm64 \
> > +            libibverbs-dev:arm64 \
> > +            libiscsi-dev:arm64 \
> > +            libjemalloc-dev:arm64 \
> > +            libjpeg62-turbo-dev:arm64 \
> > +            liblttng-ust-dev:arm64 \
> > +            liblzo2-dev:arm64 \
> > +            libncursesw5-dev:arm64 \
> > +            libnfs-dev:arm64 \
> > +            libnuma-dev:arm64 \
> > +            libpam0g-dev:arm64 \
> > +            libpixman-1-dev:arm64 \
> > +            libpng-dev:arm64 \
> > +            libpulse-dev:arm64 \
> > +            librbd-dev:arm64 \
> > +            librdmacm-dev:arm64 \
> > +            libsasl2-dev:arm64 \
> > +            libsdl2-dev:arm64 \
> > +            libsdl2-image-dev:arm64 \
> > +            libseccomp-dev:arm64 \
> > +            libselinux1-dev:arm64 \
> > +            libslirp-dev:arm64 \
> > +            libsnappy-dev:arm64 \
> > +            libspice-server-dev:arm64 \
> > +            libssh-gcrypt-dev:arm64 \
> > +            libsystemd-dev:arm64 \
> > +            libtasn1-6-dev:arm64 \
> > +            libubsan1:arm64 \
> > +            libudev-dev:arm64 \
> > +            liburing-dev:arm64 \
> > +            libusb-1.0-0-dev:arm64 \
> > +            libusbredirhost-dev:arm64 \
> > +            libvdeplug-dev:arm64 \
> > +            libvirglrenderer-dev:arm64 \
> > +            libvte-2.91-dev:arm64 \
> > +            libxen-dev:arm64 \
> > +            libzstd-dev:arm64 \
> > +            nettle-dev:arm64 \
> > +            systemtap-sdt-dev:arm64 \
> > +            xfslibs-dev:arm64 \
> > +            zlib1g-dev:arm64 && \
> > +    eatmydata apt-get autoremove -y && \
> > +    eatmydata apt-get autoclean -y && \
> > +    mkdir -p /usr/local/share/meson/cross && \
> > +    echo "[binaries]\n\
> > +c = '/usr/bin/aarch64-linux-gnu-gcc'\n\
> 
> Is it really just "c =" or should this be "cc =" ?

I believe this really does want to be 'c', as in this is the path
for the 'c' programming language  in Meson.


Regards,
Daniel
Daniel P. Berrangé Feb. 28, 2022, 9:28 a.m. UTC | #3
$SUBJECT  =~ s/lci-tool/lcitool/

On Fri, Feb 25, 2022 at 05:20:07PM +0000, Alex Bennée wrote:
> Using lci-tool update debian-arm64-cross to a Debian 11 based system.

Likewise

> As a result we can drop debian-arm64-test-cross just for building
> tests.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> Reviewed-by: Richard Henderson <richard.henderson@linaro.org>
> Message-Id: <20220211160309.335014-5-alex.bennee@linaro.org>
> ---
>  .gitlab-ci.d/container-cross.yml              |  10 +-
>  tests/docker/Makefile.include                 |   3 -
>  .../dockerfiles/debian-arm64-cross.docker     | 186 +++++++++++++++---
>  .../debian-arm64-test-cross.docker            |  13 --
>  tests/lcitool/refresh                         |  11 ++
>  tests/tcg/configure.sh                        |   2 +-
>  6 files changed, 173 insertions(+), 52 deletions(-)
>  delete mode 100644 tests/docker/dockerfiles/debian-arm64-test-cross.docker
> 



> +RUN export DEBIAN_FRONTEND=noninteractive && \
> +    apt-get update && \
> +    apt-get install -y eatmydata && \
> +    eatmydata apt-get dist-upgrade -y && \
> +    eatmydata apt-get install --no-install-recommends -y \
> +            bash \

    ...snip native packages...

> +            texinfo && \
> +    eatmydata apt-get autoremove -y && \
> +    eatmydata apt-get autoclean -y && \
> +    sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \
> +    dpkg-reconfigure locales
>  
> -RUN apt update && \
> -    DEBIAN_FRONTEND=noninteractive eatmydata \
> -    apt install -y --no-install-recommends \
> -        libbz2-dev:arm64 \
> -        liblzo2-dev:arm64 \
> -        librdmacm-dev:arm64 \
> -        libsnappy-dev:arm64 \
> -        libxen-dev:arm64
> +ENV LANG "en_US.UTF-8"
> +ENV MAKE "/usr/bin/make"
> +ENV NINJA "/usr/bin/ninja"
> +ENV PYTHON "/usr/bin/python3"
> +ENV CCACHE_WRAPPERSDIR "/usr/libexec/ccache-wrappers"
>  
> -# nettle
> -ENV QEMU_CONFIGURE_OPTS $QEMU_CONFIGURE_OPTS --enable-nettle
> +RUN export DEBIAN_FRONTEND=noninteractive && \
> +    dpkg --add-architecture arm64 && \
> +    eatmydata apt-get update && \
> +    eatmydata apt-get dist-upgrade -y && \
> +    eatmydata apt-get install --no-install-recommends -y dpkg-dev && \
> +    eatmydata apt-get install --no-install-recommends -y \
> +            g++-aarch64-linux-gnu \

  ...snip cross packages...

> +            zlib1g-dev:arm64 && \
> +    eatmydata apt-get autoremove -y && \
> +    eatmydata apt-get autoclean -y && \
> +    mkdir -p /usr/local/share/meson/cross && \
> +    echo "[binaries]\n\
> +c = '/usr/bin/aarch64-linux-gnu-gcc'\n\
> +ar = '/usr/bin/aarch64-linux-gnu-gcc-ar'\n\
> +strip = '/usr/bin/aarch64-linux-gnu-strip'\n\
> +pkgconfig = '/usr/bin/aarch64-linux-gnu-pkg-config'\n\
> +\n\
> +[host_machine]\n\
> +system = 'linux'\n\
> +cpu_family = 'aarch64'\n\
> +cpu = 'aarch64'\n\
> +endian = 'little'" > /usr/local/share/meson/cross/aarch64-linux-gnu && \
> +    dpkg-query --showformat '${Package}_${Version}_${Architecture}\n' --show > /packages.txt && \
> +    mkdir -p /usr/libexec/ccache-wrappers && \
> +    ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/aarch64-linux-gnu-c++ && \
> +    ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/aarch64-linux-gnu-cc && \
> +    ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/aarch64-linux-gnu-g++ && \
> +    ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/aarch64-linux-gnu-gcc
> +
> +ENV ABI "aarch64-linux-gnu"
> +ENV MESON_OPTS "--cross-file=aarch64-linux-gnu"
> +ENV QEMU_CONFIGURE_OPTS --cross-prefix=aarch64-linux-gnu-
> +ENV DEF_TARGET_LIST aarch64-softmmu,aarch64-linux-user

This cross dockerfile is a fully self-contained image.

Traditionally QEMU has had a split image for Debian cross targets,
where there is a base with common native packages, and then a
layer for the cross packages.

lcitool is capable of generating the image in this split format
using the arg

   --layers {all,native,foreign}

Personally I think it is simpler to just use the fully self
contained image, as it would simplify our gitlab pipeline
to only need 1 build stage for containers.  The cost is that
we'll not be sharing layers for native packages and more wall
clock time building since we're installing the same native
packages over & over.

I'm not saying to change your patch, I just wanted to point
out the possibility in case someone cares strongly about
keeping a split layer model for cross containers.

Regards,
Daniel
Alex Bennée Feb. 28, 2022, 2:39 p.m. UTC | #4
Daniel P. Berrangé <berrange@redhat.com> writes:

> $SUBJECT  =~ s/lci-tool/lcitool/
>
> On Fri, Feb 25, 2022 at 05:20:07PM +0000, Alex Bennée wrote:
>> Using lci-tool update debian-arm64-cross to a Debian 11 based system.
>
> Likewise
>
>> As a result we can drop debian-arm64-test-cross just for building
>> tests.
>> 
>> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
>> Reviewed-by: Richard Henderson <richard.henderson@linaro.org>
>> Message-Id: <20220211160309.335014-5-alex.bennee@linaro.org>
>> ---
>>  .gitlab-ci.d/container-cross.yml              |  10 +-
>>  tests/docker/Makefile.include                 |   3 -
>>  .../dockerfiles/debian-arm64-cross.docker     | 186 +++++++++++++++---
>>  .../debian-arm64-test-cross.docker            |  13 --
>>  tests/lcitool/refresh                         |  11 ++
>>  tests/tcg/configure.sh                        |   2 +-
>>  6 files changed, 173 insertions(+), 52 deletions(-)
>>  delete mode 100644 tests/docker/dockerfiles/debian-arm64-test-cross.docker
>> 
>
>
>
>> +RUN export DEBIAN_FRONTEND=noninteractive && \
>> +    apt-get update && \
>> +    apt-get install -y eatmydata && \
>> +    eatmydata apt-get dist-upgrade -y && \
>> +    eatmydata apt-get install --no-install-recommends -y \
>> +            bash \
>
>     ...snip native packages...
>
>> +            texinfo && \
>> +    eatmydata apt-get autoremove -y && \
>> +    eatmydata apt-get autoclean -y && \
>> +    sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \
>> +    dpkg-reconfigure locales
>>  
>> -RUN apt update && \
>> -    DEBIAN_FRONTEND=noninteractive eatmydata \
>> -    apt install -y --no-install-recommends \
>> -        libbz2-dev:arm64 \
>> -        liblzo2-dev:arm64 \
>> -        librdmacm-dev:arm64 \
>> -        libsnappy-dev:arm64 \
>> -        libxen-dev:arm64
>> +ENV LANG "en_US.UTF-8"
>> +ENV MAKE "/usr/bin/make"
>> +ENV NINJA "/usr/bin/ninja"
>> +ENV PYTHON "/usr/bin/python3"
>> +ENV CCACHE_WRAPPERSDIR "/usr/libexec/ccache-wrappers"
>>  
>> -# nettle
>> -ENV QEMU_CONFIGURE_OPTS $QEMU_CONFIGURE_OPTS --enable-nettle
>> +RUN export DEBIAN_FRONTEND=noninteractive && \
>> +    dpkg --add-architecture arm64 && \
>> +    eatmydata apt-get update && \
>> +    eatmydata apt-get dist-upgrade -y && \
>> +    eatmydata apt-get install --no-install-recommends -y dpkg-dev && \
>> +    eatmydata apt-get install --no-install-recommends -y \
>> +            g++-aarch64-linux-gnu \
>
>   ...snip cross packages...
>
>> +            zlib1g-dev:arm64 && \
>> +    eatmydata apt-get autoremove -y && \
>> +    eatmydata apt-get autoclean -y && \
>> +    mkdir -p /usr/local/share/meson/cross && \
>> +    echo "[binaries]\n\
>> +c = '/usr/bin/aarch64-linux-gnu-gcc'\n\
>> +ar = '/usr/bin/aarch64-linux-gnu-gcc-ar'\n\
>> +strip = '/usr/bin/aarch64-linux-gnu-strip'\n\
>> +pkgconfig = '/usr/bin/aarch64-linux-gnu-pkg-config'\n\
>> +\n\
>> +[host_machine]\n\
>> +system = 'linux'\n\
>> +cpu_family = 'aarch64'\n\
>> +cpu = 'aarch64'\n\
>> +endian = 'little'" > /usr/local/share/meson/cross/aarch64-linux-gnu && \
>> +    dpkg-query --showformat '${Package}_${Version}_${Architecture}\n' --show > /packages.txt && \
>> +    mkdir -p /usr/libexec/ccache-wrappers && \
>> +    ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/aarch64-linux-gnu-c++ && \
>> +    ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/aarch64-linux-gnu-cc && \
>> +    ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/aarch64-linux-gnu-g++ && \
>> +    ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/aarch64-linux-gnu-gcc
>> +
>> +ENV ABI "aarch64-linux-gnu"
>> +ENV MESON_OPTS "--cross-file=aarch64-linux-gnu"
>> +ENV QEMU_CONFIGURE_OPTS --cross-prefix=aarch64-linux-gnu-
>> +ENV DEF_TARGET_LIST aarch64-softmmu,aarch64-linux-user
>
> This cross dockerfile is a fully self-contained image.
>
> Traditionally QEMU has had a split image for Debian cross targets,
> where there is a base with common native packages, and then a
> layer for the cross packages.
>
> lcitool is capable of generating the image in this split format
> using the arg
>
>    --layers {all,native,foreign}
>
> Personally I think it is simpler to just use the fully self
> contained image, as it would simplify our gitlab pipeline
> to only need 1 build stage for containers.  The cost is that
> we'll not be sharing layers for native packages and more wall
> clock time building since we're installing the same native
> packages over & over.
>
> I'm not saying to change your patch, I just wanted to point
> out the possibility in case someone cares strongly about
> keeping a split layer model for cross containers.

My thinking on our layered approach has evolved over the years. One of
the problems is when the two layers get out of sync and you run into
build issues due to different states of cached layers.

I think now for full cross-builds I'm fine with just going for the full
single recipe. For cross-test building there may be more argument for
having a base layer (although in the end we also ended up with
all-test-cross because we can't have containers running containers in
GitLab).

>
> Regards,
> Daniel
Daniel P. Berrangé March 1, 2022, 10:03 a.m. UTC | #5
On Mon, Feb 28, 2022 at 02:39:17PM +0000, Alex Bennée wrote:
> 
> Daniel P. Berrangé <berrange@redhat.com> writes:
> 
> > $SUBJECT  =~ s/lci-tool/lcitool/
> >
> > On Fri, Feb 25, 2022 at 05:20:07PM +0000, Alex Bennée wrote:
> >> Using lci-tool update debian-arm64-cross to a Debian 11 based system.
> >
> > Likewise
> >
> >> As a result we can drop debian-arm64-test-cross just for building
> >> tests.
> >> 
> >> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> >> Reviewed-by: Richard Henderson <richard.henderson@linaro.org>
> >> Message-Id: <20220211160309.335014-5-alex.bennee@linaro.org>
> >> ---
> >>  .gitlab-ci.d/container-cross.yml              |  10 +-
> >>  tests/docker/Makefile.include                 |   3 -
> >>  .../dockerfiles/debian-arm64-cross.docker     | 186 +++++++++++++++---
> >>  .../debian-arm64-test-cross.docker            |  13 --
> >>  tests/lcitool/refresh                         |  11 ++
> >>  tests/tcg/configure.sh                        |   2 +-
> >>  6 files changed, 173 insertions(+), 52 deletions(-)
> >>  delete mode 100644 tests/docker/dockerfiles/debian-arm64-test-cross.docker
> >> 


> > This cross dockerfile is a fully self-contained image.
> >
> > Traditionally QEMU has had a split image for Debian cross targets,
> > where there is a base with common native packages, and then a
> > layer for the cross packages.
> >
> > lcitool is capable of generating the image in this split format
> > using the arg
> >
> >    --layers {all,native,foreign}
> >
> > Personally I think it is simpler to just use the fully self
> > contained image, as it would simplify our gitlab pipeline
> > to only need 1 build stage for containers.  The cost is that
> > we'll not be sharing layers for native packages and more wall
> > clock time building since we're installing the same native
> > packages over & over.
> >
> > I'm not saying to change your patch, I just wanted to point
> > out the possibility in case someone cares strongly about
> > keeping a split layer model for cross containers.
> 
> My thinking on our layered approach has evolved over the years. One of
> the problems is when the two layers get out of sync and you run into
> build issues due to different states of cached layers.

Oh, I'd not even thought about that possibility but yes, it makes
sense. We could have cached the base layer and when we do an
'apt-get update' in the cross layer we'll end up pulling in new
copies of packages otherwise present in the base layer, partly
defeating the point of having two layers.

Regards,
Daniel
diff mbox series

Patch

diff --git a/.gitlab-ci.d/container-cross.yml b/.gitlab-ci.d/container-cross.yml
index a3b5b90552..ed620620f8 100644
--- a/.gitlab-ci.d/container-cross.yml
+++ b/.gitlab-ci.d/container-cross.yml
@@ -21,18 +21,10 @@  amd64-debian-user-cross-container:
 
 arm64-debian-cross-container:
   extends: .container_job_template
-  stage: containers-layer2
-  needs: ['amd64-debian10-container']
+  stage: containers
   variables:
     NAME: debian-arm64-cross
 
-arm64-test-debian-cross-container:
-  extends: .container_job_template
-  stage: containers-layer2
-  needs: ['amd64-debian11-container']
-  variables:
-    NAME: debian-arm64-test-cross
-
 armel-debian-cross-container:
   extends: .container_job_template
   stage: containers-layer2
diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include
index 286f0ac5b5..eeee1e6bdf 100644
--- a/tests/docker/Makefile.include
+++ b/tests/docker/Makefile.include
@@ -137,7 +137,6 @@  DOCKER_PARTIAL_IMAGES += fedora
 endif
 
 docker-image-debian-alpha-cross: docker-image-debian10
-docker-image-debian-arm64-cross: docker-image-debian10
 docker-image-debian-armel-cross: docker-image-debian10
 docker-image-debian-armhf-cross: docker-image-debian10
 docker-image-debian-hppa-cross: docker-image-debian10
@@ -213,14 +212,12 @@  docker-image-debian-nios2-cross: $(DOCKER_FILES_DIR)/debian-toolchain.docker \
 # Specialist build images, sometimes very limited tools
 docker-image-debian-tricore-cross: docker-image-debian10
 docker-image-debian-all-test-cross: docker-image-debian10
-docker-image-debian-arm64-test-cross: docker-image-debian11
 docker-image-debian-microblaze-cross: docker-image-debian10
 docker-image-debian-nios2-cross: docker-image-debian10
 docker-image-debian-powerpc-test-cross: docker-image-debian11
 
 # These images may be good enough for building tests but not for test builds
 DOCKER_PARTIAL_IMAGES += debian-alpha-cross
-DOCKER_PARTIAL_IMAGES += debian-arm64-test-cross
 DOCKER_PARTIAL_IMAGES += debian-powerpc-test-cross
 DOCKER_PARTIAL_IMAGES += debian-hppa-cross
 DOCKER_PARTIAL_IMAGES += debian-m68k-cross debian-mips64-cross
diff --git a/tests/docker/dockerfiles/debian-arm64-cross.docker b/tests/docker/dockerfiles/debian-arm64-cross.docker
index 166e24df13..589510a7be 100644
--- a/tests/docker/dockerfiles/debian-arm64-cross.docker
+++ b/tests/docker/dockerfiles/debian-arm64-cross.docker
@@ -1,32 +1,166 @@ 
+# THIS FILE WAS AUTO-GENERATED
 #
-# Docker arm64 cross-compiler target
+#  $ lcitool dockerfile --layers all --cross aarch64 debian-11 qemu
 #
-# This docker target builds on the debian Buster base image.
-#
-FROM qemu/debian10
+# https://gitlab.com/libvirt/libvirt-ci
 
-# Add the foreign architecture we want and install dependencies
-RUN dpkg --add-architecture arm64
-RUN apt update && \
-    DEBIAN_FRONTEND=noninteractive eatmydata \
-    apt install -y --no-install-recommends \
-        crossbuild-essential-arm64
-RUN apt update && \
-    DEBIAN_FRONTEND=noninteractive eatmydata \
-    apt build-dep -yy -a arm64 --arch-only qemu
+FROM docker.io/library/debian:11-slim
 
-# Specify the cross prefix for this image (see tests/docker/common.rc)
-ENV QEMU_CONFIGURE_OPTS --cross-prefix=aarch64-linux-gnu-
-ENV DEF_TARGET_LIST aarch64-softmmu,aarch64-linux-user
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y eatmydata && \
+    eatmydata apt-get dist-upgrade -y && \
+    eatmydata apt-get install --no-install-recommends -y \
+            bash \
+            bc \
+            bsdextrautils \
+            bzip2 \
+            ca-certificates \
+            ccache \
+            dbus \
+            debianutils \
+            diffutils \
+            exuberant-ctags \
+            findutils \
+            gcovr \
+            genisoimage \
+            gettext \
+            git \
+            hostname \
+            libpcre2-dev \
+            libspice-protocol-dev \
+            libtest-harness-perl \
+            llvm \
+            locales \
+            make \
+            meson \
+            ncat \
+            ninja-build \
+            openssh-client \
+            perl-base \
+            pkgconf \
+            python3 \
+            python3-numpy \
+            python3-opencv \
+            python3-pillow \
+            python3-pip \
+            python3-sphinx \
+            python3-sphinx-rtd-theme \
+            python3-venv \
+            python3-yaml \
+            rpm2cpio \
+            sed \
+            sparse \
+            tar \
+            tesseract-ocr \
+            tesseract-ocr-eng \
+            texinfo && \
+    eatmydata apt-get autoremove -y && \
+    eatmydata apt-get autoclean -y && \
+    sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \
+    dpkg-reconfigure locales
 
-RUN apt update && \
-    DEBIAN_FRONTEND=noninteractive eatmydata \
-    apt install -y --no-install-recommends \
-        libbz2-dev:arm64 \
-        liblzo2-dev:arm64 \
-        librdmacm-dev:arm64 \
-        libsnappy-dev:arm64 \
-        libxen-dev:arm64
+ENV LANG "en_US.UTF-8"
+ENV MAKE "/usr/bin/make"
+ENV NINJA "/usr/bin/ninja"
+ENV PYTHON "/usr/bin/python3"
+ENV CCACHE_WRAPPERSDIR "/usr/libexec/ccache-wrappers"
 
-# nettle
-ENV QEMU_CONFIGURE_OPTS $QEMU_CONFIGURE_OPTS --enable-nettle
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    dpkg --add-architecture arm64 && \
+    eatmydata apt-get update && \
+    eatmydata apt-get dist-upgrade -y && \
+    eatmydata apt-get install --no-install-recommends -y dpkg-dev && \
+    eatmydata apt-get install --no-install-recommends -y \
+            g++-aarch64-linux-gnu \
+            gcc-aarch64-linux-gnu \
+            libaio-dev:arm64 \
+            libasan5:arm64 \
+            libasound2-dev:arm64 \
+            libattr1-dev:arm64 \
+            libbpf-dev:arm64 \
+            libbrlapi-dev:arm64 \
+            libbz2-dev:arm64 \
+            libc6-dev:arm64 \
+            libcacard-dev:arm64 \
+            libcap-ng-dev:arm64 \
+            libcapstone-dev:arm64 \
+            libcurl4-gnutls-dev:arm64 \
+            libdaxctl-dev:arm64 \
+            libdrm-dev:arm64 \
+            libepoxy-dev:arm64 \
+            libfdt-dev:arm64 \
+            libffi-dev:arm64 \
+            libfuse3-dev:arm64 \
+            libgbm-dev:arm64 \
+            libgcrypt20-dev:arm64 \
+            libglib2.0-dev:arm64 \
+            libglusterfs-dev:arm64 \
+            libgnutls28-dev:arm64 \
+            libgtk-3-dev:arm64 \
+            libibumad-dev:arm64 \
+            libibverbs-dev:arm64 \
+            libiscsi-dev:arm64 \
+            libjemalloc-dev:arm64 \
+            libjpeg62-turbo-dev:arm64 \
+            liblttng-ust-dev:arm64 \
+            liblzo2-dev:arm64 \
+            libncursesw5-dev:arm64 \
+            libnfs-dev:arm64 \
+            libnuma-dev:arm64 \
+            libpam0g-dev:arm64 \
+            libpixman-1-dev:arm64 \
+            libpng-dev:arm64 \
+            libpulse-dev:arm64 \
+            librbd-dev:arm64 \
+            librdmacm-dev:arm64 \
+            libsasl2-dev:arm64 \
+            libsdl2-dev:arm64 \
+            libsdl2-image-dev:arm64 \
+            libseccomp-dev:arm64 \
+            libselinux1-dev:arm64 \
+            libslirp-dev:arm64 \
+            libsnappy-dev:arm64 \
+            libspice-server-dev:arm64 \
+            libssh-gcrypt-dev:arm64 \
+            libsystemd-dev:arm64 \
+            libtasn1-6-dev:arm64 \
+            libubsan1:arm64 \
+            libudev-dev:arm64 \
+            liburing-dev:arm64 \
+            libusb-1.0-0-dev:arm64 \
+            libusbredirhost-dev:arm64 \
+            libvdeplug-dev:arm64 \
+            libvirglrenderer-dev:arm64 \
+            libvte-2.91-dev:arm64 \
+            libxen-dev:arm64 \
+            libzstd-dev:arm64 \
+            nettle-dev:arm64 \
+            systemtap-sdt-dev:arm64 \
+            xfslibs-dev:arm64 \
+            zlib1g-dev:arm64 && \
+    eatmydata apt-get autoremove -y && \
+    eatmydata apt-get autoclean -y && \
+    mkdir -p /usr/local/share/meson/cross && \
+    echo "[binaries]\n\
+c = '/usr/bin/aarch64-linux-gnu-gcc'\n\
+ar = '/usr/bin/aarch64-linux-gnu-gcc-ar'\n\
+strip = '/usr/bin/aarch64-linux-gnu-strip'\n\
+pkgconfig = '/usr/bin/aarch64-linux-gnu-pkg-config'\n\
+\n\
+[host_machine]\n\
+system = 'linux'\n\
+cpu_family = 'aarch64'\n\
+cpu = 'aarch64'\n\
+endian = 'little'" > /usr/local/share/meson/cross/aarch64-linux-gnu && \
+    dpkg-query --showformat '${Package}_${Version}_${Architecture}\n' --show > /packages.txt && \
+    mkdir -p /usr/libexec/ccache-wrappers && \
+    ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/aarch64-linux-gnu-c++ && \
+    ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/aarch64-linux-gnu-cc && \
+    ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/aarch64-linux-gnu-g++ && \
+    ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/aarch64-linux-gnu-gcc
+
+ENV ABI "aarch64-linux-gnu"
+ENV MESON_OPTS "--cross-file=aarch64-linux-gnu"
+ENV QEMU_CONFIGURE_OPTS --cross-prefix=aarch64-linux-gnu-
+ENV DEF_TARGET_LIST aarch64-softmmu,aarch64-linux-user
diff --git a/tests/docker/dockerfiles/debian-arm64-test-cross.docker b/tests/docker/dockerfiles/debian-arm64-test-cross.docker
deleted file mode 100644
index 53a9012beb..0000000000
--- a/tests/docker/dockerfiles/debian-arm64-test-cross.docker
+++ /dev/null
@@ -1,13 +0,0 @@ 
-#
-# Docker arm64 cross-compiler target (tests only)
-#
-# This docker target builds on the debian Bullseye base image.
-#
-FROM qemu/debian11
-
-# Add the foreign architecture we want and install dependencies
-RUN dpkg --add-architecture arm64
-RUN apt update && \
-    DEBIAN_FRONTEND=noninteractive eatmydata \
-        apt install -y --no-install-recommends \
-        crossbuild-essential-arm64 gcc-10-aarch64-linux-gnu
diff --git a/tests/lcitool/refresh b/tests/lcitool/refresh
index 4ab90a310a..ada73f7045 100755
--- a/tests/lcitool/refresh
+++ b/tests/lcitool/refresh
@@ -76,6 +76,12 @@  ubuntu2004_tsanhack = [
    "RUN sed -i 's/^const/static const/g' /usr/lib/llvm-10/lib/clang/10.0.0/include/sanitizer/tsan_interface.h\n"
 ]
 
+def debian_cross_build(prefix, targets):
+   conf = "ENV QEMU_CONFIGURE_OPTS --cross-prefix=%s\n" % (prefix)
+   targets = "ENV DEF_TARGET_LIST %s\n" % (targets)
+   return "".join([conf, targets])
+
+
 try:
    generate_dockerfile("centos8", "centos-stream-8")
    generate_dockerfile("fedora", "fedora-35")
@@ -86,6 +92,11 @@  try:
    generate_dockerfile("opensuse-leap", "opensuse-leap-152")
    generate_dockerfile("alpine", "alpine-edge")
 
+   generate_dockerfile("debian-arm64-cross", "debian-11",
+                       cross="aarch64",
+                       trailer=debian_cross_build("aarch64-linux-gnu-",
+                                                  "aarch64-softmmu,aarch64-linux-user"))
+
    generate_cirrus("freebsd-12")
    generate_cirrus("freebsd-13")
    generate_cirrus("macos-11")
diff --git a/tests/tcg/configure.sh b/tests/tcg/configure.sh
index 763e9b6ad8..adc95d6a44 100755
--- a/tests/tcg/configure.sh
+++ b/tests/tcg/configure.sh
@@ -97,7 +97,7 @@  for target in $target_list; do
     aarch64-*)
       # We don't have any bigendian build tools so we only use this for AArch64
       container_hosts="x86_64 aarch64"
-      container_image=debian-arm64-test-cross
+      container_image=debian-arm64-cross
       container_cross_cc=aarch64-linux-gnu-gcc-10
       ;;
     alpha-*)