From f85160ab3383fee91d3bf6e4a38bfbd62f27627e Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Thu, 2 May 2024 14:37:50 +0900 Subject: [PATCH] Support reproducible builds (except packages) See docker-library/official-images issue 16044 - `SOURCE_DATE_EPOCH` is added. The value is consumed by the build scripts to make the binary reproducible. - `/tmp/*` is removed as they contain files created by `memcached` - For Debian, `/var/log/*` is removed as they contain timestamps - For Debian, `/var/cache/ldconfig/aux-cache` is removed as they contain inode numbers, etc. - For Alpine, virtual package versions are pinned to "0" to eliminate the timestamp-based version numbers that appear in `/etc/apk/world` and `/lib/apk/db/installed` > [!NOTE] > The following topics are NOT covered by this commit: > > - To reproduce file timestamps in layers, BuildKit has to be executed with > `--output type=,rewrite-timestamp=true`. > Needs BuildKit v0.13 or later. > > - To reproduce the base image by the hash, reproducers may: > - modify the `FROM` instruction in Dockerfile manually > - or, use the `CONVERT` action of source policies to replace the base image. > > > - To reproduce packages, see the `RUN` instruction hook proposed in > moby/buildkit issue 4576 Signed-off-by: Akihiro Suda --- 1/alpine/Dockerfile | 17 ++++++++++++++--- 1/debian/Dockerfile | 21 +++++++++++++++++++-- Dockerfile.template | 25 +++++++++++++++++++++---- 3 files changed, 54 insertions(+), 9 deletions(-) diff --git a/1/alpine/Dockerfile b/1/alpine/Dockerfile index 57a42c1..c37f302 100644 --- a/1/alpine/Dockerfile +++ b/1/alpine/Dockerfile @@ -6,7 +6,12 @@ FROM alpine:3.20 +# The global SOURCE_DATE_EPOCH is consumed by commands that are not associated with a source artifact. +# This is not propagated from --build-arg: https://github.com/moby/buildkit/issues/4576#issuecomment-2159501282 +ENV SOURCE_DATE_EPOCH 0 + # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added +# On Debian, useradd recognizes SOURCE_DATE_EPOCH to reproduce the "lastchanged" field in /etc/shadow. RUN set -eux; \ addgroup -g 11211 memcache; \ adduser -D -u 11211 -G memcache memcache @@ -20,7 +25,7 @@ ENV MEMCACHED_SHA1 85e2cb9520beba71d7fc69f5717208a57facde28 RUN set -eux; \ \ - apk add --no-cache --virtual .build-deps \ + apk add --no-cache --virtual .build-deps=0 \ ca-certificates \ coreutils \ cyrus-sasl-dev \ @@ -46,6 +51,10 @@ RUN set -eux; \ cd /usr/src/memcached; \ \ gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \ + SOURCE_DATE_EPOCH="$(find . -type f -exec stat -c '%Y' {} + | sort -nr | head -n1)"; \ + export SOURCE_DATE_EPOCH; \ +# for logging validation/edification + date --date "@$SOURCE_DATE_EPOCH" --rfc-2822; \ ./configure \ --build="$gnuArch" \ --enable-extstore \ @@ -69,10 +78,12 @@ RUN set -eux; \ | sort -u \ | awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \ )"; \ - apk add --no-network --virtual .memcached-rundeps $runDeps; \ + apk add --no-network --virtual .memcached-rundeps=0 $runDeps; \ apk del --no-network .build-deps; \ \ - memcached -V + memcached -V ;\ +# clean up for reproducibility + rm -rf /tmp/* COPY docker-entrypoint.sh /usr/local/bin/ RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh # backwards compat diff --git a/1/debian/Dockerfile b/1/debian/Dockerfile index 0a11a22..731f361 100644 --- a/1/debian/Dockerfile +++ b/1/debian/Dockerfile @@ -6,7 +6,12 @@ FROM debian:bookworm-slim +# The global SOURCE_DATE_EPOCH is consumed by commands that are not associated with a source artifact. +# This is not propagated from --build-arg: https://github.com/moby/buildkit/issues/4576#issuecomment-2159501282 +ENV SOURCE_DATE_EPOCH 0 + # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added +# On Debian, useradd recognizes SOURCE_DATE_EPOCH to reproduce the "lastchanged" field in /etc/shadow. RUN set -eux; \ groupadd --system --gid 11211 memcache; \ useradd --system --gid memcache --uid 11211 memcache @@ -17,7 +22,9 @@ RUN set -eux; \ apt-get install -y --no-install-recommends \ libsasl2-modules \ ; \ - rm -rf /var/lib/apt/lists/* + rm -rf /var/lib/apt/lists/*; \ +# clean up for reproducibility + rm -rf /var/log/* /var/cache/ldconfig/aux-cache ENV MEMCACHED_VERSION 1.6.31 ENV MEMCACHED_URL https://memcached.org/files/memcached-1.6.31.tar.gz @@ -41,6 +48,8 @@ RUN set -eux; \ wget \ ; \ rm -rf /var/lib/apt/lists/*; \ +# clean up for reproducibility + rm -rf /var/log/* /var/cache/ldconfig/aux-cache; \ \ wget -O memcached.tar.gz "$MEMCACHED_URL"; \ echo "$MEMCACHED_SHA1 memcached.tar.gz" | sha1sum -c -; \ @@ -51,6 +60,10 @@ RUN set -eux; \ cd /usr/src/memcached; \ \ gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \ + SOURCE_DATE_EPOCH="$(find . -type f -exec stat -c '%Y' {} + | sort -nr | head -n1)"; \ + export SOURCE_DATE_EPOCH; \ +# for logging validation/edification + date --date "@$SOURCE_DATE_EPOCH" --rfc-2822; \ ./configure \ --build="$gnuArch" \ --enable-extstore \ @@ -82,8 +95,12 @@ RUN set -eux; \ | xargs -r apt-mark manual \ ; \ apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ +# clean up for reproducibility + rm -rf /var/log/* /var/cache/ldconfig/aux-cache; \ \ - memcached -V + memcached -V ;\ +# clean up for reproducibility + rm -rf /tmp/* COPY docker-entrypoint.sh /usr/local/bin/ RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh # backwards compat diff --git a/Dockerfile.template b/Dockerfile.template index e509d4b..99659b8 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -4,7 +4,12 @@ FROM alpine:{{ .alpine.version }} FROM debian:{{ .debian.version }}-slim {{ ) end -}} +# The global SOURCE_DATE_EPOCH is consumed by commands that are not associated with a source artifact. +# This is not propagated from --build-arg: https://github.com/moby/buildkit/issues/4576#issuecomment-2159501282 +ENV SOURCE_DATE_EPOCH 0 + # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added +# On Debian, useradd recognizes SOURCE_DATE_EPOCH to reproduce the "lastchanged" field in /etc/shadow. {{ if env.variant == "alpine" then ( -}} RUN set -eux; \ addgroup -g 11211 memcache; \ @@ -24,7 +29,9 @@ RUN set -eux; \ apt-get install -y --no-install-recommends \ libsasl2-modules \ ; \ - rm -rf /var/lib/apt/lists/* + rm -rf /var/lib/apt/lists/*; \ +# clean up for reproducibility + rm -rf /var/log/* /var/cache/ldconfig/aux-cache {{ ) end -}} ENV MEMCACHED_VERSION {{ .version }} @@ -34,7 +41,7 @@ ENV MEMCACHED_SHA1 {{ .sha1 }} RUN set -eux; \ \ {{ if env.variant == "alpine" then ( -}} - apk add --no-cache --virtual .build-deps \ + apk add --no-cache --virtual .build-deps=0 \ ca-certificates \ coreutils \ cyrus-sasl-dev \ @@ -67,6 +74,8 @@ RUN set -eux; \ wget \ ; \ rm -rf /var/lib/apt/lists/*; \ +# clean up for reproducibility + rm -rf /var/log/* /var/cache/ldconfig/aux-cache; \ {{ ) end -}} \ wget -O memcached.tar.gz "$MEMCACHED_URL"; \ @@ -78,6 +87,10 @@ RUN set -eux; \ cd /usr/src/memcached; \ \ gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \ + SOURCE_DATE_EPOCH="$(find . -type f -exec stat -c '%Y' {} + | sort -nr | head -n1)"; \ + export SOURCE_DATE_EPOCH; \ +# for logging validation/edification + date --date "@$SOURCE_DATE_EPOCH" --rfc-2822; \ ./configure \ --build="$gnuArch" \ --enable-extstore \ @@ -109,7 +122,7 @@ RUN set -eux; \ | sort -u \ | awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \ )"; \ - apk add --no-network --virtual .memcached-rundeps $runDeps; \ + apk add --no-network --virtual .memcached-rundeps=0 $runDeps; \ apk del --no-network .build-deps; \ {{ ) else ( -}} apt-mark auto '.*' > /dev/null; \ @@ -123,9 +136,13 @@ RUN set -eux; \ | xargs -r apt-mark manual \ ; \ apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ +# clean up for reproducibility + rm -rf /var/log/* /var/cache/ldconfig/aux-cache; \ {{ ) end -}} \ - memcached -V + memcached -V ;\ +# clean up for reproducibility + rm -rf /tmp/* COPY docker-entrypoint.sh /usr/local/bin/ RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh # backwards compat