pax_global_header00006660000000000000000000000064134576733370014534gustar00rootroot0000000000000052 comment=20b4f32e93b9f0b039e2fd9fefd8527a5cb21b3e debuerreotype-0.10/000077500000000000000000000000001345767333700143325ustar00rootroot00000000000000debuerreotype-0.10/.dockerignore000066400000000000000000000000271345767333700170050ustar00rootroot00000000000000** !VERSION !scripts/ debuerreotype-0.10/.travis.sh000077500000000000000000000012761345767333700162650ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail epoch="$(TZ=UTC date --date "$TIMESTAMP" +%s)" serial="$(TZ=UTC date --date "@$epoch" +%Y%m%d)" buildArgs=() if [ "$SUITE" = 'eol' ]; then buildArgs+=( '--eol' ) SUITE="$CODENAME" elif [ -n "${CODENAME:-}" ]; then buildArgs+=( '--codename-copy' ) fi if [ -n "${ARCH:-}" ]; then buildArgs+=( "--arch=${ARCH}" ) if [ "$ARCH" != 'i386' ]; then buildArgs+=( '--qemu' ) fi fi buildArgs+=( travis "$SUITE" "@$epoch" ) checkFile="travis/$serial/${ARCH:-amd64}/${CODENAME:-$SUITE}/rootfs.tar.xz" set -x ./scripts/debuerreotype-version ./build.sh "${buildArgs[@]}" real="$(sha256sum "$checkFile" | cut -d' ' -f1)" [ -z "$SHA256" ] || [ "$SHA256" = "$real" ] debuerreotype-0.10/.travis.yml000066400000000000000000000055351345767333700164530ustar00rootroot00000000000000language: bash services: docker env: - SUITE=stable CODENAME=jessie TIMESTAMP=2017-01-01T00:00:00Z SHA256=55ba54fdca819df18d813be36503b0a02abf1570c3bf5999b10891ccca5448e2 - SUITE=jessie CODENAME= TIMESTAMP=2017-01-01T00:00:00Z SHA256=55ba54fdca819df18d813be36503b0a02abf1570c3bf5999b10891ccca5448e2 - SUITE=testing CODENAME=stretch TIMESTAMP=2017-01-01T00:00:00Z SHA256=1608c820c1d9c9d8adf210f80b1d751e5c26179aa27a1c1ddb8e41ae0222d8c4 - SUITE=stretch CODENAME= TIMESTAMP=2017-01-01T00:00:00Z SHA256=1608c820c1d9c9d8adf210f80b1d751e5c26179aa27a1c1ddb8e41ae0222d8c4 - SUITE=unstable CODENAME=sid TIMESTAMP=2017-01-01T00:00:00Z SHA256=49a5152822ec9f0e1a61ff1d02671681f12fc1aba083f39e972f6ff897b69c80 - SUITE=sid CODENAME= TIMESTAMP=2017-01-01T00:00:00Z SHA256=49a5152822ec9f0e1a61ff1d02671681f12fc1aba083f39e972f6ff897b69c80 - SUITE=oldstable CODENAME=wheezy TIMESTAMP=2017-01-01T00:00:00Z SHA256=f1bd72548e3c25ce222fb9e2bb57a5b6d4b01042180894fb05d83a0251e6dab1 - SUITE=wheezy CODENAME= TIMESTAMP=2017-01-01T00:00:00Z SHA256=f1bd72548e3c25ce222fb9e2bb57a5b6d4b01042180894fb05d83a0251e6dab1 # EOL suites testing - SUITE=eol CODENAME=etch TIMESTAMP=2017-01-01T00:00:00Z SHA256=b48e999ab4fda1720b0dc863d38cdd4d6b55530f34f262a28949eb6173102da9 - SUITE=eol CODENAME=lenny TIMESTAMP=2017-01-01T00:00:00Z SHA256=1a2fffd34daa4a6bb968aebe86480a4093035a23700ec5f2e883423b9b4dcfa7 - SUITE=eol CODENAME=woody ARCH=i386 TIMESTAMP=2017-01-01T00:00:00Z SHA256=ef4bc81e31db51fa9f095811ddbcc8a005f05f098596317d5a138fa90157bf40 # qemu-debootstrap testing - ARCH=arm64 SUITE=jessie CODENAME= TIMESTAMP=2017-01-01T00:00:00Z SHA256=893efc1b9db1ba2df4f171d4422194a408f9810d3b55d9b0cd66fcc7722f7567 # a few entries for "today" to try and catch issues like https://github.com/debuerreotype/debuerreotype/issues/41 sooner - SUITE=unstable CODENAME= TIMESTAMP="today 00:00:00" SHA256= - SUITE=stable CODENAME= TIMESTAMP="today 00:00:00" SHA256= - SUITE=oldstable CODENAME= TIMESTAMP="today 00:00:00" SHA256= addons: apt: packages: - binfmt-support - qemu-user-static before_script: - docker run -d --name squignix --restart always tianon/squignix # TODO temporary!! (once https://github.com/tianon/pgp-happy-eyeballs/tree/travis-squignix is deleted, this should be) -- squignix is necessary for building etch and woody who otherwise are so poorly behaved they get rate limited by snapshot.d.o (https://travis-ci.org/debuerreotype/debuerreotype/builds/479633791) - wget -qO- https://github.com/tianon/pgp-happy-eyeballs/raw/713f2a81bf3eac1752f0c41b271444f5b57e93c9/hack-my-builds.sh | bash script: - travis_retry ./.travis.sh after_script: - docker images - docker logs rawdns - docker logs squignix # TODO temporary!! (see above) debuerreotype-0.10/Dockerfile000066400000000000000000000030431345767333700163240ustar00rootroot00000000000000# docker run --cap-add SYS_ADMIN --cap-drop SETFCAP --tmpfs /tmp:dev,exec,suid,noatime ... # bootstrapping a new architecture? # ./scripts/debuerreotype-init /tmp/docker-rootfs stretch now # ./scripts/debuerreotype-minimizing-config /tmp/docker-rootfs # ./scripts/debuerreotype-debian-sources-list /tmp/docker-rootfs stretch # ./scripts/debuerreotype-tar /tmp/docker-rootfs - | docker import - debian:stretch-slim # alternate: # debootstrap --variant=minbase stretch /tmp/docker-rootfs # tar -cC /tmp/docker-rootfs . | docker import - debian:stretch-slim # (or your own favorite set of "debootstrap" commands to create a base image for building this one FROM) FROM debian:stretch-slim RUN apt-get update && apt-get install -y --no-install-recommends \ debootstrap \ wget ca-certificates \ xz-utils \ \ gnupg dirmngr \ && rm -rf /var/lib/apt/lists/* # see ".dockerignore" COPY . /opt/debuerreotype RUN set -ex; \ cd /opt/debuerreotype/scripts; \ for f in debuerreotype-*; do \ ln -svL "$PWD/$f" "/usr/local/bin/$f"; \ done; \ version="$(debuerreotype-version)"; \ [ "$version" != 'unknown' ]; \ echo "debuerreotype version $version" WORKDIR /tmp # a few example md5sum values for amd64: # debuerreotype-init test-stretch stretch 2017-05-08T00:00:00Z # debuerreotype-tar test-stretch test-stretch.tar # md5sum test-stretch.tar # 14206d5b9b2991e98f5214c3d310e4fa # debuerreotype-init test-jessie jessie 2017-05-08T00:00:00Z # debuerreotype-tar test-jessie test-jessie.tar # md5sum test-jessie.tar # 57f98d3636000630080e5ba208508e10 debuerreotype-0.10/LICENSE000066400000000000000000000020601345767333700153350ustar00rootroot00000000000000Copyright 2017 Tianon Gravi Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. debuerreotype-0.10/README.md000066400000000000000000000147441345767333700156230ustar00rootroot00000000000000# Debuerreotype [![Build Status](https://travis-ci.org/debuerreotype/debuerreotype.svg?branch=master)](https://travis-ci.org/debuerreotype/debuerreotype/branches) Reproducible, [snapshot](http://snapshot.debian.org)-based Debian rootfs builds (especially for Docker). This is based on [lamby](https://github.com/lamby)'s work for reproducible `debootstrap`: - https://github.com/lamby/debootstrap/commit/66b15380814aa62ca4b5807270ac57a3c8a0558d - https://wiki.debian.org/ReproducibleInstalls ### "Debuerreotype"? The name is an attempt at riffing off the photography basis of the word "snapshot". The [daguerreotype](https://en.wikipedia.org/wiki/Daguerreotype) process was an early method for taking photographs, and this is a method for taking "photographs" of Debian at a given point in time. ## Why? The goal is to create an auditable, reproducible process for creating rootfs tarballs (especially for use in Docker) of Debian releases, based on point-in-time snapshots from [snapshot.debian.org](http://snapshot.debian.org). However, as noted below, the only strictly Docker-specific script is `debuerreotype-minimizing-config`, which applies many configuration tweaks which are useful for Docker users and may or may not be useful outside of that context. ## Usage The usage of the scripts here center around a "rootfs" directory, which is both the working directory for building the target rootfs, and contains the `debuerreotype-epoch` file, which records our snapshot.debian.org epoch value (so we can adjust timestamps using it, as it is the basis for our reproducibility). Available scripts: | *script* | *purpose* | | --- | --- | | `debuerreotype-init` | create the initial "rootfs", given a suite and a timestamp (in some format `date(1)` can parse); `sources.list` will be pointing at snapshot.debian.org | | `debuerreotype-chroot` | run a command in the given "rootfs" (using `unshare` to mount `/dev`, `/proc`, and `/sys` from the parent environment in a simple, safe way) | | `debuerreotype-apt-get` | run `apt-get` via `debuerreotype-chroot`, including `-o Acquire::Check-Valid-Until=false` to account for older snapshots with (now) invalid `Valid-Until` values | | `debuerreotype-minimizing-config` | apply configuration tweaks to make the rootfs minimal and keep it minimal (especially targeted at Docker images, with comments explicitly describing Docker use cases) | | `debuerreotype-slimify` | remove files such as documentation to create an even smaller rootfs (used for creating `slim` variants of the Docker images, for example) | | `debuerreotype-debian-sources-list` | generate an appropriate Debian `sources.list` in the rootfs given a suite (especially for updating `sources.list` to point at deb.debian.org before generating outputs) | | `debuerreotype-fixup` | invoked by `debuerreotype-tar` to fixup timestamps and remove known-bad log files for determinism | | `debuerreotype-tar` | deterministically create a tar file of the rootfs | | `debuerreotype-version` | print out the version of the current `debuerreotype` installation | A simple `Dockerfile` is provided for using these scripts in a simple deterministic environment based on Docker, but given a recent enough version of `debootstrap`, they should run fine outside Docker as well (and their deterministic properties have been verified on at least a Gentoo host in addition to the provided Debian-based Docker environment). The provided `Dockerfile` also includes comments with hints for bootstrapping the environment on a new architecture (which then presumably doesn't have a `debian` Docker base image yet). Full example: (see [`build.sh`](build.sh) for this in practice) ```console $ debuerreotype-init rootfs stretch 2017-01-01T00:00:00Z I: Retrieving InRelease I: Checking Release signature I: Valid Release signature (key id 126C0D24BD8A2942CC7DF8AC7638D0442B90D010) ... I: Checking component main on http://snapshot.debian.org/archive/debian/20170101T000000Z... ... I: Base system installed successfully. $ cat rootfs/debuerreotype-epoch 1483228800 $ debuerreotype-minimizing-config rootfs $ debuerreotype-apt-get rootfs update -qq $ debuerreotype-apt-get rootfs dist-upgrade -yqq $ debuerreotype-apt-get rootfs install -yqq --no-install-recommends inetutils-ping iproute2 debconf: delaying package configuration, since apt-utils is not installed Selecting previously unselected package libelf1:amd64. (Reading database ... 6299 files and directories currently installed.) Preparing to unpack .../0-libelf1_0.166-2.2_amd64.deb ... Unpacking libelf1:amd64 (0.166-2.2) ... Selecting previously unselected package libmnl0:amd64. Preparing to unpack .../1-libmnl0_1.0.4-2_amd64.deb ... Unpacking libmnl0:amd64 (1.0.4-2) ... Selecting previously unselected package iproute2. Preparing to unpack .../2-iproute2_4.9.0-1_amd64.deb ... Unpacking iproute2 (4.9.0-1) ... Selecting previously unselected package netbase. Preparing to unpack .../3-netbase_5.3_all.deb ... Unpacking netbase (5.3) ... Selecting previously unselected package inetutils-ping. Preparing to unpack .../4-inetutils-ping_2%3a1.9.4-2+b1_amd64.deb ... Unpacking inetutils-ping (2:1.9.4-2+b1) ... Setting up libelf1:amd64 (0.166-2.2) ... Processing triggers for libc-bin (2.24-8) ... Setting up libmnl0:amd64 (1.0.4-2) ... Setting up netbase (5.3) ... Setting up inetutils-ping (2:1.9.4-2+b1) ... Setting up iproute2 (4.9.0-1) ... Processing triggers for libc-bin (2.24-8) ... $ debuerreotype-debian-sources-list rootfs stretch $ debuerreotype-tar rootfs - | sha256sum a076d4cd04f68ee117e598a40cc947ad051fc8b063340da015fdceddeb1b0e75 - $ # try it! you should get that same sha256sum value! ``` ## How much have you verified this? Well, I ran the scripts across seven explicit architectures (`amd64`, `arm64`, `armel`, `armhf`, `i386`, `ppc64el`, `s390x`) and eight explicit suites (`oldstable`, `stable`, `testing`, `unstable`, `wheezy`, `jessie`, `stretch`, `sid`) for a timestamp of `2017-05-16T00:00:00Z` (where supported, since `wheezy`/`oldstable` didn't or no longer currently supports some of those architectures), and there were no modifications to any of the tarballs after several runs across several days. Additionally, Travis runs with a fixed timestamp value across several suites to verify that their checksums are reproducible, as expected. From time to time, comments in the files generated by `debuerreotype-minimizing-config` might change (for example), which would obviously result in a different checksum, but a simple [`diffoscope`](https://diffoscope.org/) should be sufficient to verify that the change is benign. debuerreotype-0.10/VERSION000066400000000000000000000000051345767333700153750ustar00rootroot000000000000000.10 debuerreotype-0.10/build-all.sh000077500000000000000000000034701345767333700165420ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail suites=( unstable testing stable oldstable oldoldstable # just in case (will no-op with "not supported on 'arch'" unless it exists) oldoldoldstable ) thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/scripts/.constants.sh" \ --flags 'no-build' \ -- \ '[--no-build] ' \ 'output 2017-05-08T00:00:00Z' eval "$dgetopt" build=1 while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --no-build) build= ;; # for skipping "docker build" --) break ;; *) eusage "unknown flag '$flag'" ;; esac done outputDir="${1:-}"; shift || eusage 'missing output-dir' timestamp="${1:-}"; shift || eusage 'missing timestamp' mkdir -p "$outputDir" outputDir="$(readlink -f "$outputDir")" ver="$("$thisDir/scripts/debuerreotype-version")" ver="${ver%% *}" dockerImage="debuerreotype/debuerreotype:$ver" [ -z "$build" ] || docker build -t "$dockerImage" "$thisDir" mirror="$("$thisDir/scripts/.snapshot-url.sh" "$timestamp")" secmirror="$("$thisDir/scripts/.snapshot-url.sh" "$timestamp" 'debian-security')" dpkgArch="$(docker run --rm "$dockerImage" dpkg --print-architecture | awk -F- '{ print $NF }')" echo echo "-- BUILDING TARBALLS FOR '$dpkgArch' FROM '$mirror/' --" echo for suite in "${suites[@]}"; do doSkip= case "$suite" in testing|unstable) ;; *) if ! wget --quiet --spider "$secmirror/dists/$suite/updates/main/binary-$dpkgArch/Packages.gz"; then doSkip=1 fi ;; esac if ! wget --quiet --spider "$mirror/dists/$suite/main/binary-$dpkgArch/Packages.gz"; then doSkip=1 fi if [ -n "$doSkip" ]; then echo >&2 echo >&2 "warning: '$suite' not supported on '$dpkgArch' (at '$timestamp'); skipping" echo >&2 continue fi "$thisDir/build.sh" --no-build --codename-copy "$outputDir" "$suite" "$timestamp" done debuerreotype-0.10/build.sh000077500000000000000000000303641345767333700157760ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/scripts/.constants.sh" \ --flags 'no-build,codename-copy' \ --flags 'eol,arch:,qemu' \ -- \ '[--no-build] [--codename-copy] [--eol] [--arch=] [--qemu] ' \ 'output stretch 2017-05-08T00:00:00Z --codename-copy output stable 2017-05-08T00:00:00Z --eol output squeeze 2016-03-14T00:00:00Z --eol --arch i386 output sarge 2016-03-14T00:00:00Z' \ eval "$dgetopt" build=1 codenameCopy= eol= arch= qemu= while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --no-build) build= ;; # for skipping "docker build" --codename-copy) codenameCopy=1 ;; # for copying a "stable.tar.xz" to "stretch.tar.xz" with updated sources.list (saves a lot of extra building work) --eol) eol=1 ;; # for using "archive.debian.org" --arch) arch="$1"; shift ;; # for adding "--arch" to debuerreotype-init --qemu) qemu=1 ;; # for using "qemu-debootstrap" --) break ;; *) eusage "unknown flag '$flag'" ;; esac done outputDir="${1:-}"; shift || eusage 'missing output-dir' suite="${1:-}"; shift || eusage 'missing suite' timestamp="${1:-}"; shift || eusage 'missing timestamp' mkdir -p "$outputDir" outputDir="$(readlink -f "$outputDir")" securityArgs=( --cap-add SYS_ADMIN --cap-drop SETFCAP ) if docker info | grep -q apparmor; then # AppArmor blocks mount :) securityArgs+=( --security-opt apparmor=unconfined ) fi if [ "$suite" = 'potato' ]; then # --debian-eol potato wants to run "chroot ... mount ... /proc" which gets blocked (i386, ancient binaries, blah blah blah) securityArgs+=( --security-opt seccomp=unconfined ) fi ver="$("$thisDir/scripts/debuerreotype-version")" ver="${ver%% *}" dockerImage="debuerreotype/debuerreotype:$ver" [ -z "$build" ] || docker build -t "$dockerImage" "$thisDir" if [ -n "$qemu" ]; then [ -z "$build" ] || docker build -t "$dockerImage-qemu" - <<-EODF FROM $dockerImage RUN apt-get update && apt-get install -y --no-install-recommends qemu-user-static && rm -rf /var/lib/apt/lists/* EODF dockerImage="$dockerImage-qemu" fi docker run \ --rm \ "${securityArgs[@]}" \ --tmpfs /tmp:dev,exec,suid,noatime \ -w /tmp \ -e suite="$suite" \ -e timestamp="$timestamp" \ -e codenameCopy="$codenameCopy" \ -e eol="$eol" -e arch="$arch" -e qemu="$qemu" \ -e TZ='UTC' -e LC_ALL='C' \ --hostname debuerreotype \ "$dockerImage" \ bash -Eeuo pipefail -c ' set -x epoch="$(date --date "$timestamp" +%s)" serial="$(date --date "@$epoch" +%Y%m%d)" dpkgArch="${arch:-$(dpkg --print-architecture | awk -F- "{ print \$NF }")}" exportDir="output" outputDir="$exportDir/$serial/$dpkgArch/$suite" touch_epoch() { while [ "$#" -gt 0 ]; do local f="$1"; shift touch --no-dereference --date="@$epoch" "$f" done } debuerreotypeScriptsDir="$(dirname "$(readlink -f "$(which debuerreotype-init)")")" for archive in "" security; do if [ -z "$eol" ]; then snapshotUrl="$("$debuerreotypeScriptsDir/.snapshot-url.sh" "@$epoch" "${archive:+debian-${archive}}")" else snapshotUrl="$("$debuerreotypeScriptsDir/.snapshot-url.sh" "@$epoch" "debian-archive")/debian${archive:+-${archive}}" fi snapshotUrlFile="$exportDir/$serial/$dpkgArch/snapshot-url${archive:+-${archive}}" mkdir -p "$(dirname "$snapshotUrlFile")" echo "$snapshotUrl" > "$snapshotUrlFile" touch_epoch "$snapshotUrlFile" done export GNUPGHOME="$(mktemp -d)" keyring="$GNUPGHOME/debian-archive-$suite-keyring.gpg" if [ "$suite" = potato ]; then # src:debian-archive-keyring was created in 2006, thus does not include a key for potato gpg --batch --no-default-keyring --keyring "$keyring" \ --keyserver ha.pool.sks-keyservers.net \ --recv-keys 8FD47FF1AA9372C37043DC28AA7DEB7B722F1AED else # check against all releases (ie, combine both "debian-archive-keyring.gpg" and "debian-archive-removed-keys.gpg"), since we cannot really know whether the target release became EOL later than the snapshot date we are targeting gpg --batch --no-default-keyring --keyring "$keyring" --import \ /usr/share/keyrings/debian-archive-keyring.gpg \ /usr/share/keyrings/debian-archive-removed-keys.gpg fi snapshotUrl="$(< "$exportDir/$serial/$dpkgArch/snapshot-url")" mkdir -p "$outputDir" wget -O "$outputDir/Release.gpg" "$snapshotUrl/dists/$suite/Release.gpg" wget -O "$outputDir/Release" "$snapshotUrl/dists/$suite/Release" gpgv \ --keyring "$keyring" \ "$outputDir/Release.gpg" \ "$outputDir/Release" codename="$(awk -F ": " "\$1 == \"Codename\" { print \$2; exit }" "$outputDir/Release")" if [ -n "$codenameCopy" ] && [ "$codename" = "$suite" ]; then # if codename already is the same as suite, then making a copy does not make any sense codenameCopy= fi if [ -n "$codenameCopy" ] && [ -z "$codename" ]; then echo >&2 "error: --codename-copy specified but we failed to get a Codename for $suite" exit 1 fi { initArgs=( --arch="$dpkgArch" ) if [ -z "$eol" ]; then initArgs+=( --debian ) else initArgs+=( --debian-eol ) fi initArgs+=( --keyring "$keyring" ) # disable merged-usr (for now?) due to the following compelling arguments: # - https://bugs.debian.org/src:usrmerge ("dpkg-query" breaks, etc) # - https://bugs.debian.org/914208 ("buildd" variant disables merged-usr still) # - https://github.com/debuerreotype/docker-debian-artifacts/issues/60#issuecomment-461426406 initArgs+=( --no-merged-usr ) if [ -n "$qemu" ]; then initArgs+=( --debootstrap="qemu-debootstrap" ) fi debuerreotype-init "${initArgs[@]}" rootfs "$suite" "@$epoch" if [ -n "$eol" ]; then debuerreotype-gpgv-ignore-expiration-config rootfs fi debuerreotype-minimizing-config rootfs debuerreotype-apt-get rootfs update -qq debuerreotype-apt-get rootfs dist-upgrade -yqq aptVersion="$("$debuerreotypeScriptsDir/.apt-version.sh" rootfs)" if dpkg --compare-versions "$aptVersion" ">=" "0.7.14~"; then # https://salsa.debian.org/apt-team/apt/commit/06d79436542ccf3e9664306da05ba4c34fba4882 noInstallRecommends="--no-install-recommends" else # --debian-eol etch and lower do not support --no-install-recommends noInstallRecommends="-o APT::Install-Recommends=0" fi if [ -n "$eol" ] && dpkg --compare-versions "$aptVersion" ">=" "0.7.26~"; then # https://salsa.debian.org/apt-team/apt/commit/1ddb859611d2e0f3d9ea12085001810f689e8c99 echo "Acquire::Check-Valid-Until \"false\";" > rootfs/etc/apt/apt.conf.d/check-valid-until.conf # TODO make this a real script so it can have a nice comment explaining why we do it for EOL releases? fi # make a couple copies of rootfs so we can create other variants for variant in slim sbuild; do mkdir "rootfs-$variant" tar -cC rootfs . | tar -xC "rootfs-$variant" done # prefer iproute2 if it exists iproute=iproute2 if ! debuerreotype-apt-get rootfs install -qq -s iproute2 &> /dev/null; then # poor wheezy iproute=iproute fi ping=iputils-ping if debuerreotype-chroot rootfs bash -c "command -v ping > /dev/null"; then # if we already have "ping" (as in --debian-eol potato), skip installing any extra ping package ping= fi debuerreotype-apt-get rootfs install -y $noInstallRecommends $ping $iproute debuerreotype-slimify rootfs-slim # this should match the list added to the "buildd" variant in debootstrap and the list installed by sbuild # https://anonscm.debian.org/cgit/d-i/debootstrap.git/tree/scripts/sid?id=706a45681c5bba5e062a9b02e19f079cacf2a3e8#n26 # https://anonscm.debian.org/cgit/buildd-tools/sbuild.git/tree/bin/sbuild-createchroot?id=eace3d3e59e48d26eaf069d9b63a6a4c868640e6#n194 debuerreotype-apt-get rootfs-sbuild install -y $noInstallRecommends build-essential fakeroot create_artifacts() { local targetBase="$1"; shift local rootfs="$1"; shift local suite="$1"; shift local variant="$1"; shift # make a copy of the snapshot-facing sources.list file before we overwrite it cp "$rootfs/etc/apt/sources.list" "$targetBase.sources-list-snapshot" touch_epoch "$targetBase.sources-list-snapshot" local tarArgs=() if [ -n "$qemu" ]; then tarArgs+=( --exclude="./usr/bin/qemu-*-static" ) fi if [ "$variant" != "sbuild" ]; then debuerreotype-debian-sources-list $([ -z "$eol" ] || echo "--eol") "$rootfs" "$suite" else # sbuild needs "deb-src" entries debuerreotype-debian-sources-list --deb-src $([ -z "$eol" ] || echo "--eol") "$rootfs" "$suite" # APT has odd issues with "Acquire::GzipIndexes=false" + "file://..." sources sometimes # (which are used in sbuild for "--extra-package") # Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) # ... # E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) rm -f "$rootfs/etc/apt/apt.conf.d/docker-gzip-indexes" # TODO figure out the bug and fix it in APT instead /o\ # schroot is picky about "/dev" (which is excluded by default in "debuerreotype-tar") # see https://github.com/debuerreotype/debuerreotype/pull/8#issuecomment-305855521 tarArgs+=( --include-dev ) fi case "$suite" in sarge) # for some reason, sarge creates "/var/cache/man/index.db" with some obvious embedded unix timestamps (but if we exclude it, "man" still works properly, so *shrug*) tarArgs+=( --exclude ./var/cache/man/index.db ) ;; woody) # woody not only contains "exim", but launches it during our build process and tries to email "root@debuerreotype" (which fails and creates non-reproducibility) tarArgs+=( --exclude ./var/spool/exim --exclude ./var/log/exim ) ;; potato) tarArgs+=( # for some reason, pototo leaves a core dump (TODO figure out why??) --exclude "./core" --exclude "./qemu*.core" # also, it leaves some junk in /tmp (/tmp/fdmount.conf.tmp.XXX) --exclude "./tmp/fdmount.conf.tmp.*" ) ;; esac debuerreotype-tar "${tarArgs[@]}" "$rootfs" "$targetBase.tar.xz" du -hsx "$targetBase.tar.xz" sha256sum "$targetBase.tar.xz" | cut -d" " -f1 > "$targetBase.tar.xz.sha256" touch_epoch "$targetBase.tar.xz.sha256" debuerreotype-chroot "$rootfs" bash -c " if ! dpkg-query -W 2> /dev/null; then # --debian-eol woody has no dpkg-query dpkg -l fi " > "$targetBase.manifest" echo "$epoch" > "$targetBase.debuerreotype-epoch" debuerreotype-version > "$targetBase.debuerreotype-version" touch_epoch "$targetBase.manifest" "$targetBase.debuerreotype-epoch" "$targetBase.debuerreotype-version" for f in debian_version os-release apt/sources.list; do targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")" if [ -e "$rootfs/etc/$f" ]; then # /etc/os-release does not exist in --debian-eol squeeze, for example (hence the existence check) cp "$rootfs/etc/$f" "$targetFile" touch_epoch "$targetFile" fi done } for rootfs in rootfs*/; do rootfs="${rootfs%/}" # "rootfs", "rootfs-slim", ... du -hsx "$rootfs" variant="${rootfs#rootfs}" # "", "-slim", ... variant="${variant#-}" # "", "slim", ... variantDir="$outputDir/$variant" mkdir -p "$variantDir" targetBase="$variantDir/rootfs" create_artifacts "$targetBase" "$rootfs" "$suite" "$variant" done if [ -n "$codenameCopy" ]; then codenameDir="$exportDir/$serial/$dpkgArch/$codename" mkdir -p "$codenameDir" tar -cC "$outputDir" --exclude="**/rootfs.*" . | tar -xC "$codenameDir" for rootfs in rootfs*/; do rootfs="${rootfs%/}" # "rootfs", "rootfs-slim", ... variant="${rootfs#rootfs}" # "", "-slim", ... variant="${variant#-}" # "", "slim", ... variantDir="$codenameDir/$variant" targetBase="$variantDir/rootfs" # point sources.list back at snapshot.debian.org temporarily (but this time pointing at $codename instead of $suite) debuerreotype-debian-sources-list --snapshot $([ -z "$eol" ] || echo "--eol") "$rootfs" "$codename" create_artifacts "$targetBase" "$rootfs" "$codename" "$variant" done fi } >&2 tar -cC "$exportDir" . ' | tar -xvC "$outputDir" debuerreotype-0.10/raspbian.sh000077500000000000000000000130431345767333700164710ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/scripts/.constants.sh" \ --flags 'no-build' \ -- \ '[--no-build] ' \ 'output stretch' eval "$dgetopt" build=1 while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --no-build) build= ;; # for skipping "docker build" --) break ;; *) eusage "unknown flag '$flag'" ;; esac done outputDir="${1:-}"; shift || eusage 'missing output-dir' suite="${1:-}"; shift || eusage 'missing suite' mkdir -p "$outputDir" outputDir="$(readlink -f "$outputDir")" securityArgs=( --cap-add SYS_ADMIN --cap-drop SETFCAP ) if docker info | grep -q apparmor; then # AppArmor blocks mount :) securityArgs+=( --security-opt apparmor=unconfined ) fi ver="$("$thisDir/scripts/debuerreotype-version")" ver="${ver%% *}" dockerImage="debuerreotype/debuerreotype:$ver" [ -z "$build" ] || docker build -t "$dockerImage" "$thisDir" raspbianDockerImage="$dockerImage-raspbian" [ -z "$build" ] || docker build -t "$raspbianDockerImage" - <<-EODF FROM $dockerImage RUN wget -O raspbian.deb 'https://archive.raspbian.org/raspbian/pool/main/r/raspbian-archive-keyring/raspbian-archive-keyring-udeb_20120528.2_all.udeb' \\ && apt install -y ./raspbian.deb \\ && rm raspbian.deb EODF docker run \ --rm \ "${securityArgs[@]}" \ -v /tmp \ -w /tmp \ -e suite="$suite" \ -e TZ='UTC' -e LC_ALL='C' \ "$raspbianDockerImage" \ bash -Eeuo pipefail -c ' set -x mirror="http://archive.raspbian.org/raspbian" dpkgArch="armhf" exportDir="output" outputDir="$exportDir/raspbian/$dpkgArch/$suite" mkdir -p "$outputDir" wget -O "$outputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg" wget -O "$outputDir/Release" "$mirror/dists/$suite/Release" gpgv \ --keyring /usr/share/keyrings/raspbian-archive-keyring.gpg \ "$outputDir/Release.gpg" \ "$outputDir/Release" { debuerreotype-init --non-debian \ --arch "$dpkgArch" \ --keyring /usr/share/keyrings/raspbian-archive-keyring.gpg \ --no-merged-usr \ rootfs "$suite" "$mirror" epoch="$(< rootfs/debuerreotype-epoch)" touch_epoch() { while [ "$#" -gt 0 ]; do local f="$1"; shift touch --no-dereference --date="@$epoch" "$f" done } debuerreotype-minimizing-config rootfs debuerreotype-apt-get rootfs update -qq debuerreotype-apt-get rootfs dist-upgrade -yqq # make a couple copies of rootfs so we can create other variants for variant in slim sbuild; do mkdir "rootfs-$variant" tar -cC rootfs . | tar -xC "rootfs-$variant" done # prefer iproute2 if it exists iproute=iproute2 if ! debuerreotype-chroot rootfs apt-get install -qq -s iproute2 &> /dev/null; then # poor wheezy iproute=iproute fi debuerreotype-apt-get rootfs install -y --no-install-recommends iputils-ping $iproute debuerreotype-slimify rootfs-slim # this should match the list added to the "buildd" variant in debootstrap and the list installed by sbuild # https://anonscm.debian.org/cgit/d-i/debootstrap.git/tree/scripts/sid?id=706a45681c5bba5e062a9b02e19f079cacf2a3e8#n26 # https://anonscm.debian.org/cgit/buildd-tools/sbuild.git/tree/bin/sbuild-createchroot?id=eace3d3e59e48d26eaf069d9b63a6a4c868640e6#n194 debuerreotype-apt-get rootfs-sbuild install -y --no-install-recommends build-essential fakeroot create_artifacts() { local targetBase="$1"; shift local rootfs="$1"; shift local suite="$1"; shift local variant="$1"; shift if [ "$variant" != "sbuild" ]; then debuerreotype-tar "$rootfs" "$targetBase.tar.xz" else # sbuild needs "deb-src" entries debuerreotype-chroot "$rootfs" sed -ri -e "/^deb / p; s//deb-src /" /etc/apt/sources.list # APT has odd issues with "Acquire::GzipIndexes=false" + "file://..." sources sometimes # (which are used in sbuild for "--extra-package") # Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) # ... # E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) rm -f "$rootfs/etc/apt/apt.conf.d/docker-gzip-indexes" # TODO figure out the bug and fix it in APT instead /o\ # schroot is picky about "/dev" (which is excluded by default in "debuerreotype-tar") # see https://github.com/debuerreotype/debuerreotype/pull/8#issuecomment-305855521 debuerreotype-tar --include-dev "$rootfs" "$targetBase.tar.xz" fi du -hsx "$targetBase.tar.xz" sha256sum "$targetBase.tar.xz" | cut -d" " -f1 > "$targetBase.tar.xz.sha256" touch_epoch "$targetBase.tar.xz.sha256" debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest" echo "$epoch" > "$targetBase.debuerreotype-epoch" touch_epoch "$targetBase.manifest" "$targetBase.debuerreotype-epoch" for f in debian_version os-release apt/sources.list; do targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")" cp "$rootfs/etc/$f" "$targetFile" touch_epoch "$targetFile" done } for rootfs in rootfs*/; do rootfs="${rootfs%/}" # "rootfs", "rootfs-slim", ... du -hsx "$rootfs" variant="${rootfs#rootfs}" # "", "-slim", ... variant="${variant#-}" # "", "slim", ... variantDir="$outputDir/$variant" mkdir -p "$variantDir" targetBase="$variantDir/rootfs" create_artifacts "$targetBase" "$rootfs" "$suite" "$variant" done } >&2 tar -cC "$exportDir" . ' | tar -xvC "$outputDir" debuerreotype-0.10/scripts/000077500000000000000000000000001345767333700160215ustar00rootroot00000000000000debuerreotype-0.10/scripts/.apt-version.sh000077500000000000000000000016001345767333700207020ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ 'rootfs' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] # scrape our APT version so we can do some basic feature detection (especially to remove unsupported settings on --debian-eol) "$thisDir/debuerreotype-chroot" "$targetDir" bash -c ' if command -v dpkg-query &> /dev/null; then dpkg-query --show --showformat "\${Version}\n" apt else # if dpkg-query does not exist, we must be on woody or potato, so just assume something ancient like 0.5.4 (since that is what woody includes and is old enough to cover all our features being excluded) echo 0.5.4 fi ' debuerreotype-0.10/scripts/.constants.sh000066400000000000000000000036171345767333700204560ustar00rootroot00000000000000#!/usr/bin/env bash # constants of the universe export TZ='UTC' LC_ALL='C' umask 0002 scriptsDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" self="$(basename "$0")" options="$(getopt -n "$BASH_SOURCE" -o '+' --long 'flags:,flags-short:' -- "$@")" dFlags='help,version' dFlagsShort='h?' usageStr= __cgetopt() { eval "set -- $options" # in a function since otherwise "set" will overwrite the parent script's positional args too unset options while true; do local flag="$1"; shift case "$flag" in --flags) dFlags="${dFlags:+$dFlags,}$1"; shift ;; --flags-short) dFlagsShort="${dFlagsShort}$1"; shift ;; --) break ;; *) echo >&2 "error: unexpected $BASH_SOURCE flag '$flag'"; exit 1 ;; esac done while [ "$#" -gt 0 ]; do local IFS=$'\n' local usagePrefix='usage:' usageLine= for usageLine in $1; do usageStr+="$usagePrefix $self${usageLine:+ $usageLine}"$'\n' usagePrefix=' ' done usagePrefix=' ie:' for usageLine in $2; do usageStr+="$usagePrefix $self${usageLine:+ $usageLine}"$'\n' usagePrefix=' ' done usageStr+=$'\n' shift 2 done } __cgetopt _version() { local v if [ -r "$scriptsDir/../VERSION" ]; then v="$(< "$scriptsDir/../VERSION")" else v='unknown' fi if [ -d "$scriptsDir/../.git" ] && command -v git > /dev/null; then local commit="$(git -C "$scriptsDir" rev-parse --short 'HEAD^{commit}')" v="$v commit $commit" fi echo "$v" } usage() { echo -n "$usageStr" local v="$(_version)" echo "debuerreotype version $v" } eusage() { if [ "$#" -gt 0 ]; then echo >&2 "error: $*"$'\n' fi usage >&2 exit 1 } _dgetopt() { getopt -n "$self" \ -o "+$dFlagsShort" \ --long "$dFlags" \ -- "$@" \ || eusage 'getopt failed' } dgetopt='options="$(_dgetopt "$@")"; eval "set -- $options"; unset options' dgetopt-case() { local flag="$1"; shift case "$flag" in -h|'-?'|--help) usage; exit 0 ;; --version) _version; exit 0 ;; esac } debuerreotype-0.10/scripts/.dpkg-arch.sh000077500000000000000000000011041345767333700202720ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ 'rootfs' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] arch="$("$thisDir/debuerreotype-chroot" "$targetDir" dpkg --print-architecture)" # --debian-eol woody likes to give us "i386-none" arch="${arch%-none}" echo "$arch" | awk -F- '{ print $NF }' debuerreotype-0.10/scripts/.fix-apt-comments.sh000077500000000000000000000013311345767333700216270ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ ' [file ...]' \ '0.7.22 rootfs/etc/apt/apt.conf.d/example' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done aptVersion="${1:-}"; shift || eusage 'missing apt-version' [ "$#" -gt 0 ] || eusage 'missing file(s)' # support for "apt.conf" comments of the style "# xxx" was added in 0.7.22 # (https://salsa.debian.org/apt-team/apt/commit/81e9789b12374073e848c73c79e235f82c14df44) if dpkg --compare-versions "$aptVersion" '>=' '0.7.22~'; then exit fi sed -ri -e 's!^#!//!' "$@" debuerreotype-0.10/scripts/.gpgv-ignore-expiration.sh000077500000000000000000000025621345767333700230470ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeu # For the sake of EOL releases (whose archive keys have often expired), we need a fake "gpgv" substitute that will essentially ignore *just* key expiration. # (So we get *some* signature validation instead of using something like "--allow-unauthenticated" or "--force-yes" which disable security entirely instead.) # Intended usage (APT >= 1.1): # apt-get -o Apt::Key::gpgvcommand=/.../.debuerreotype-gpgv-ignore-expiration ... # or (APT < 1.1): # apt-get -o Dir::Bin::gpg=/.../.debuerreotype-gpgv-ignore-expiration ... # (https://salsa.debian.org/apt-team/apt/commit/12841e8320aa499554ac50b102b222900bb1b879) # Functionally, this script will scrape "--status-fd" (which is the only way a user of "gpgv" can care about / process expired key metadata) and MITM "gpgv" to replace EXPKEYSIG with GOODSIG instead. _status_fd() { while [ "$#" -gt 0 ]; do case "$1" in --status-fd) echo "$2" return 0 ;; esac shift done return 1 } if fd="$(_status_fd "$@")" && [ -n "$fd" ]; then # older bash (3.2, lenny) doesn't support variable file descriptors (hence "eval") # (bash: syntax error near unexpected token `$fd') eval 'exec gpgv "$@" '"$fd"'> >(sed "s/EXPKEYSIG/GOODSIG/" >&'"$fd"')' fi # no "--status-fd"? no worries! ("gpgv" without "--status-fd" doesn't seem to care about expired keys, so we don't have to either) exec gpgv "$@" debuerreotype-0.10/scripts/.slimify-excludes000066400000000000000000000012201345767333700213030ustar00rootroot00000000000000# This file contains the list of files/directories which will be removed for "slim" image variants. # https://github.com/tianon/docker-brew-debian/issues/48 # https://wiki.ubuntu.com/ReducingDiskFootprint#Drop_unnecessary_files /usr/share/doc/* /usr/share/groff/* /usr/share/info/* /usr/share/linda/* /usr/share/lintian/overrides/* /usr/share/locale/* /usr/share/man/* # https://anonscm.debian.org/cgit/collab-maint/localepurge.git/tree/usr/share/localepurge/gen-dpkg-cfg.pl?id=b32c8d46d43be2027096f5a202ac789a637ceb39#n9 /usr/share/locale/* /usr/share/gnome/help/*/* /usr/share/doc/kde/HTML/*/* /usr/share/omf/*/*-*.emf # see also .slimify-includes debuerreotype-0.10/scripts/.slimify-includes000066400000000000000000000012431345767333700213020ustar00rootroot00000000000000# This file contains the list of files/directories which will *NOT* be removed for "slim" image variants. # https://github.com/tianon/docker-brew-debian/issues/48 # https://wiki.ubuntu.com/ReducingDiskFootprint#Drop_unnecessary_files /usr/share/doc/*/copyright # https://anonscm.debian.org/cgit/collab-maint/localepurge.git/tree/usr/share/localepurge/gen-dpkg-cfg.pl?id=b32c8d46d43be2027096f5a202ac789a637ceb39#n9 /usr/share/locale/locale.alias /usr/share/gnome/help/*/C/* /usr/share/doc/kde/HTML/C/* /usr/share/omf/*/*-C.emf /usr/share/locale/languages /usr/share/locale/all_languages /usr/share/locale/currency/* /usr/share/locale/l10n/* # see also .slimify-excludes debuerreotype-0.10/scripts/.snapshot-url.sh000077500000000000000000000010241345767333700210720ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ ' [archive]' \ '2017-05-08T00:00:00Z debian-security' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done timestamp="${1:-}"; shift || eusage 'missing timestamp' archive="${1:-debian}" t="$(date --date "$timestamp" '+%Y%m%dT%H%M%SZ')" echo "http://snapshot.debian.org/archive/$archive/$t" debuerreotype-0.10/scripts/.tar-exclude000066400000000000000000000033321345767333700202400ustar00rootroot00000000000000# the file we store the "epoch" of a given rootfs in ./debuerreotype-epoch # "/dev" is a special case in "debuerreotype-tar" #./dev/** ./proc/** ./sys/** # targeted exclusions to get rid of everything except "/var/cache/apt/archives/partial" and "/var/lib/apt/lists/partial" # (https://salsa.debian.org/apt-team/apt/commit/1cd1c398d18b78f4aa9d882a5de5385f4538e0be) ./var/cache/apt/*.bin ./var/cache/apt/archives/*.deb ./var/cache/apt/archives/lock ./var/lib/apt/lists/*Packages* ./var/lib/apt/lists/*Release* ./var/lib/apt/lists/lock # https://salsa.debian.org/apt-team/apt/commit/5555ef9850b7e66aa02d39bb7d624fdf3e43edb2 (APT 0.9.14 removed support for /var/state/apt) ./var/state/apt/lists/*Packages* ./var/state/apt/lists/*Release* ./var/state/apt/lists/lock # ends up with host-kernel info ./etc/apt/apt.conf.d/01autoremove-kernels # useful data in these, but includes timestamps too ./var/log/apt/history.log ./var/log/apt/term.log # wheezy-only file which contains host-kernel info ./run/motd.dynamic # wheezy-only file which APT doesn't even use (and has timestamp embedded in binary data) # see https://github.com/debuerreotype/debuerreotype/issues/7 ./etc/apt/trustdb.gpg # a wheezy-only file which only shows up when building via debootstrap in buster+ for some reason ./run/shm/.run-transition # Debian creates this file reproducibly, but Ubuntu does not # (according to "man 1 journalctl", this is automatically recreated by "journalctl --update-catalog") # Tails also removes this file to achieve reproducibility (https://labs.riseup.net/code/projects/tails/repository/revisions/b1e05c8aac12fc79293f6a220b40a538d4f38c51/diff/config/chroot_local-hooks/99-zzzzzz_reproducible-builds-post-processing) ./var/lib/systemd/catalog/database debuerreotype-0.10/scripts/debuerreotype-apt-get000077500000000000000000000011031345767333700221570ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ ' arguments' \ 'rootfs update' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] epoch="$(< "$targetDir/debuerreotype-epoch")" export SOURCE_DATE_EPOCH="$epoch" "$thisDir/debuerreotype-chroot" "$targetDir" apt-get -o Acquire::Check-Valid-Until=false "$@" debuerreotype-0.10/scripts/debuerreotype-chroot000077500000000000000000000020431345767333700221200ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ ' [args...]' \ 'rootfs apt-get update' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' cmd="${1:-}"; shift || eusage 'missing command' [ -n "$targetDir" ] epoch="$(< "$targetDir/debuerreotype-epoch")" [ -n "$epoch" ] export targetDir epoch unshare --mount bash -Eeuo pipefail -c ' [ -n "$targetDir" ] # just to be safe for dir in dev proc sys; do if [ -e "$targetDir/$dir" ]; then # --debian-eol woody and below have no /sys mount --rbind "/$dir" "$targetDir/$dir" fi done mount --rbind --read-only /etc/resolv.conf "$targetDir/etc/resolv.conf" exec chroot "$targetDir" /usr/bin/env -i PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" TZ="$TZ" LC_ALL="$LC_ALL" SOURCE_DATE_EPOCH="$epoch" "$@" ' -- "$cmd" "$@" debuerreotype-0.10/scripts/debuerreotype-debian-sources-list000077500000000000000000000070241345767333700245020ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ --flags 'eol,snapshot' \ --flags 'deb-src' \ -- \ '[--deb-src] [--eol] [--no-snapshot] ' \ '--snapshot rootfs stretch --eol rootfs wheezy' eval "$dgetopt" eol= snapshot= debSrc= while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --eol) eol=1 ;; --snapshot) snapshot=1 ;; --deb-src) debSrc=1 ;; --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' suite="${1:-}"; shift || eusage 'missing suite' [ -n "$targetDir" ] epoch="$(< "$targetDir/debuerreotype-epoch")" standardMirror='http://deb.debian.org/debian' snapshotStandardMirrors=( "$("$thisDir/.snapshot-url.sh" "@$epoch")" ) securityMirror='http://security.debian.org/debian-security' snapshotSecurityMirrors=( "$("$thisDir/.snapshot-url.sh" "@$epoch" 'debian-security')" ) if [ -n "$eol" ]; then archiveSnapshotMirror="$("$thisDir/.snapshot-url.sh" "@$epoch" 'debian-archive')" standardMirror='http://archive.debian.org/debian' snapshotStandardMirrors=( "$archiveSnapshotMirror/debian" "${snapshotStandardMirrors[@]}" ) securityMirror='http://archive.debian.org/debian-security' snapshotSecurityMirrors=( "$archiveSnapshotMirror/debian-security" "${snapshotSecurityMirrors[@]}" ) fi comp='main' arch="$("$thisDir/.dpkg-arch.sh" "$targetDir")" deb() { local suite="$1"; shift local comp="$1"; shift local target="$1"; shift # "standard" or "security" local nonSnapshotMirror= snapshotMirrors=() case "$target" in standard) nonSnapshotMirror="$standardMirror"; snapshotMirrors=( "${snapshotStandardMirrors[@]}" ) ;; security) nonSnapshotMirror="$securityMirror"; snapshotMirrors=( "${snapshotSecurityMirrors[@]}" ) ;; *) echo >&2 "error: unknown 'deb' line target: '$target'"; exit 1 ;; esac local found= mirror for mirror in "${snapshotMirrors[@]}"; do # http://snapshot.debian.org/archive/debian-archive/20160314T000000Z/debian/dists/squeeze-updates/main/binary-amd64/Packages.gz if wget --quiet --spider -O /dev/null -o /dev/null "$mirror/dists/$suite/$comp/binary-$arch/Packages.gz"; then found="$mirror" break fi done if [ -z "$found" ]; then echo >&2 "warning: no apparent '$suite/$comp' for '$arch' on any of the following; skipping" for mirror in "${snapshotMirrors[@]}"; do echo >&2 " - $mirror"; done return fi if [ -n "$snapshot" ]; then mirror="$found" else echo "# deb $found $suite $comp" mirror="$nonSnapshotMirror" fi echo "deb $mirror $suite $comp" if [ -n "$debSrc" ]; then echo "deb-src $mirror $suite $comp" fi } # https://github.com/tianon/go-aptsources/blob/e066ed9cd8cd9eef7198765bd00ec99679e6d0be/target.go#L16-L58 { case "$suite" in sid|unstable|testing) deb "$suite" "$comp" standard ;; *) # https://salsa.debian.org/installer-team/apt-setup/tree/d7a642fb5fc76e4f0b684db53984bdb9123f8360/generators deb "$suite" "$comp" standard # "50mirror" deb "$suite/updates" "$comp" security # "91security" deb "$suite-updates" "$comp" standard # "92updates" # https://wiki.debian.org/SourcesList#Example_sources.list if [ "$suite" = 'squeeze' ]; then # https://wiki.debian.org/DebianSqueeze#FAQ deb "$suite-lts" "$comp" standard fi ;; esac } > "$targetDir/etc/apt/sources.list" chmod 0644 "$targetDir/etc/apt/sources.list" if [ ! -s "$targetDir/etc/apt/sources.list" ]; then echo >&2 "error: sources.list ended up empty -- something is definitely wrong" exit 1 fi debuerreotype-0.10/scripts/debuerreotype-fixup000077500000000000000000000020061345767333700217540ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ 'rootfs' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] epoch="$(< "$targetDir/debuerreotype-epoch")" [ -n "$epoch" ] # https://github.com/lamby/debootstrap/commit/66b15380814aa62ca4b5807270ac57a3c8a0558d#diff-de4eef4ab836e5c6c9c1f820a2f624baR709 rm -f \ "$targetDir/var/log/dpkg.log" \ "$targetDir/var/log/bootstrap.log" \ "$targetDir/var/log/alternatives.log" \ "$targetDir/var/cache/ldconfig/aux-cache" # https://github.com/debuerreotype/debuerreotype/pull/32 rm -f "$targetDir/run/mount/utab" # (also remove the directory, but only if it's empty) rmdir "$targetDir/run/mount" 2>/dev/null || : find "$targetDir" \ -newermt "@$epoch" \ -exec touch --no-dereference --date="@$epoch" '{}' + debuerreotype-0.10/scripts/debuerreotype-gpgv-ignore-expiration-config000077500000000000000000000035561345767333700265030ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ 'rootfs' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] aptVersion="$("$thisDir/.apt-version.sh" "$targetDir")" # if we're on APT 0.6 or lower, this isn't relevant # (added in 0.7.21 / 0.7.20.2+lenny1; https://salsa.debian.org/apt-team/apt/commit/0b77f4775db7bc45964e0337b8978a170b3f0483) if dpkg --compare-versions "$aptVersion" '<<' '0.7.20~'; then echo >&2 "note: skipping $self: APT version ($aptVersion) too old to be relevant" exit fi sourceFile="$thisDir/.gpgv-ignore-expiration.sh" targetPath='/usr/local/bin/.debuerreotype-gpgv-ignore-expiration' targetFile="$targetDir$targetPath" cp -T "$sourceFile" "$targetFile" chmod 0755 "$targetFile" # APT 1.1+ changed to use "apt-key verify" instead of invoking "gpgv" directly # (https://salsa.debian.org/apt-team/apt/commit/12841e8320aa499554ac50b102b222900bb1b879) aptConfigKey='Apt::Key::gpgvcommand' case "$aptVersion" in 0.* | 1.0*) aptConfigKey='Dir::Bin::gpg' ;; esac cat > "$targetDir/etc/apt/apt.conf.d/debuerreotype-gpgv-ignore-expiration" <<-EOF # For the sake of EOL releases (whose archive keys have often expired), we need # a fake "gpgv" substitute that will essentially ignore *just* key expiration. # (So we get *some* signature validation instead of using something like # "--allow-unauthenticated" or "--force-yes" which disable security entirely # instead.) $aptConfigKey "$targetPath"; EOF chmod 0644 "$targetDir/etc/apt/apt.conf.d/debuerreotype-gpgv-ignore-expiration" "$thisDir/.fix-apt-comments.sh" "$aptVersion" "$targetDir/etc/apt/apt.conf.d/debuerreotype-gpgv-ignore-expiration" debuerreotype-0.10/scripts/debuerreotype-init000077500000000000000000000136151345767333700215740ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ --flags 'debian,debian-eol,non-debian' \ --flags 'debootstrap:' \ --flags 'debootstrap-script:' \ --flags 'keyring:,arch:,include:,exclude:' \ --flags 'merged-usr,no-merged-usr' \ -- \ ' ' \ 'rootfs stretch 2017-05-08T00:00:00Z --debian-eol rootfs squeeze 2016-03-14T00:00:00Z' \ \ '--non-debian [--debootstrap-script=xyz] ' \ '--non-debian rootfs xenial http://archive.ubuntu.com/ubuntu' eval "$dgetopt" nonDebian= debianEol= debootstrap= script= keyring= arch= include= exclude= noMergedUsr= while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --debian) nonDebian= ; debianEol= ;; --debian-eol) nonDebian= ; debianEol=1 ;; --non-debian) nonDebian=1; debianEol= ;; --debootstrap) debootstrap="$1"; shift ;; --debootstrap-script) script="$1"; shift ;; --keyring) keyring="$1"; shift ;; --arch) arch="$1"; shift ;; --include) include="${include:+$include,}$1"; shift ;; --exclude) exclude="${exclude:+$exclude,}$1"; shift ;; --merged-usr) noMergedUsr= ;; --no-merged-usr) noMergedUsr=1 ;; --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] || eusage 'target-dir required' # must be non-empty if [ -e "$targetDir" ] && [ -z "$(find "$targetDir" -maxdepth 0 -empty)" ]; then echo >&2 "error: '$targetDir' already exists (and isn't empty)!" exit 1 fi suite="${1:-}"; shift || eusage 'missing suite' timestamp= mirror= if [ -z "$nonDebian" ]; then timestamp="${1:-}"; shift || eusage 'missing timestamp' else mirror="${1:-}"; shift || eusage 'missing mirror' timestamp="$(wget -qO- "$mirror/dists/$suite/Release" | awk -F ': ' '$1 == "Date" { print $2 }')" fi epoch="$(date --date "$timestamp" '+%s')" export SOURCE_DATE_EPOCH="$epoch" if [ -z "$nonDebian" ]; then if [ -z "$debianEol" ]; then mirror="$("$thisDir/.snapshot-url.sh" "@$epoch")" else mirrorbase="$("$thisDir/.snapshot-url.sh" "@$epoch" 'debian-archive')" mirror="$mirrorbase/debian" fi fi debootstrapArgs=( --force-check-gpg ) minbaseSupported="$( scriptFile="$( if [ -n "$script" ]; then readlink -f "$script" else cd /usr/share/debootstrap/scripts readlink -f "$suite" fi )" if grep -q 'minbase' "$scriptFile"; then echo 1 fi )" if [ -n "$minbaseSupported" ]; then # --debian-eol sarge and older do not support minbase debootstrapArgs+=( --variant=minbase ) fi [ -n "$noMergedUsr" ] && debootstrapArgs+=( --no-merged-usr ) || debootstrapArgs+=( --merged-usr ) [ -z "$keyring" ] || debootstrapArgs+=( --keyring="$keyring" ) [ -z "$arch" ] || debootstrapArgs+=( --arch="$arch" ) [ -z "$include" ] || debootstrapArgs+=( --include="$include" ) [ -z "$exclude" ] || debootstrapArgs+=( --exclude="$exclude" ) debootstrapArgs+=( "$suite" "$targetDir" "$mirror" ) [ -z "$script" ] || debootstrapArgs+=( "$script" ) : "${debootstrap:=debootstrap}" if ! "$debootstrap" "${debootstrapArgs[@]}"; then if [ -f "$targetDir/debootstrap/debootstrap.log" ]; then echo >&2 echo >&2 "error: '$debootstrap' failed!" echo >&2 echo >&2 ' Full command:' echo >&2 echo >&2 " $(printf ' %q' "$debootstrap" "${debootstrapArgs[@]}")" echo >&2 echo >&2 ' Logs:' echo >&2 cat >&2 "$targetDir/debootstrap/debootstrap.log" echo >&2 fi exit 1 fi echo "$epoch" > "$targetDir/debuerreotype-epoch" if [ -z "$nonDebian" ]; then "$thisDir/debuerreotype-debian-sources-list" --snapshot $([ -z "$debianEol" ] || echo '--eol') "$targetDir" "$suite" "$thisDir/debuerreotype-apt-get" "$targetDir" update -qq fi # since we're minbase, we know everything included is either essential, or a dependency of essential, so let's get clean "apt-mark showmanual" output "$thisDir/debuerreotype-chroot" "$targetDir" bash -c ' # --debian-eol squeeze and below do not have python in minbase, thus "apt-mark" fails to run # bash: /usr/bin/apt-mark: /usr/bin/python: bad interpreter: No such file or directory # (also, squeeze APT does not treat essential packages as special, and will offer to purge them if they get marked as auto-installed) if apt-mark --help &> /dev/null; then apt-mark auto ".*" > /dev/null if [ -n "$1" ]; then # if the user asked for anything to be included extra (like "xyz-archive-keyring"), mark those packages as manually installed IFS=","; includePackages=( $1 ); unset IFS apt-mark manual "${includePackages[@]}" fi fi ' -- "$include" echo 'debuerreotype' > "$targetDir/etc/hostname" echo "$epoch" \ | md5sum \ | cut -f1 -d' ' \ > "$targetDir/etc/machine-id" # TODO should we only do this if "/etc/machine-id" already exists? { echo '# https://1.1.1.1 (privacy-focused, highly-available DNS service)' echo 'nameserver 1.1.1.1' echo 'nameserver 1.0.0.1' } > "$targetDir/etc/resolv.conf" chmod 0644 \ "$targetDir/etc/hostname" \ "$targetDir/etc/machine-id" \ "$targetDir/etc/resolv.conf" # fix ownership/permissions on / (otherwise "debootstrap" leaves them as-is which causes reproducibility issues) chown 0:0 "$targetDir" chmod 0755 "$targetDir" # https://bugs.debian.org/857803 # adjust field 3 in /etc/shadow and /etc/shadow- to $(( epoch / 60 / 60 / 24 )), if it's larger sp_lstchg="$(( epoch / 60 / 60 / 24 ))" for shadowFile in etc/shadow etc/shadow-; do # --debian-eol etch and older do not include /etc/shadow- [ -e "$targetDir/$shadowFile" ] || continue newShadowFile="$shadowFile.debuerreotype" awk -F ':' \ -v OFS=':' \ -v sp_lstchg="$sp_lstchg" \ '{ if ($3 > sp_lstchg) { $3 = sp_lstchg } print }' "$targetDir/$shadowFile" > "$targetDir/$newShadowFile" if [ "$(< "$targetDir/$shadowFile")" != "$(< "$targetDir/$newShadowFile")" ]; then # use "cat" instead of "mv" so permissions don't change cat "$targetDir/$newShadowFile" > "$targetDir/$shadowFile" fi rm -f "$targetDir/$newShadowFile" done debuerreotype-0.10/scripts/debuerreotype-minimizing-config000077500000000000000000000161641345767333700242500ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ 'rootfs' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] aptVersion="$("$thisDir/.apt-version.sh" "$targetDir")" # https://github.com/docker/docker/blob/d6f4fe9e38b60f63e429fff7ffced9c26cbf8236/contrib/mkimage/debootstrap#L63-L177 # prevent init scripts from running during install/update cat > "$targetDir/usr/sbin/policy-rc.d" <<-'EOF' #!/bin/sh # For most Docker users, "apt-get install" only happens during "docker build", # where starting services doesn't work and often fails in humorous ways. This # prevents those failures by stopping the services from attempting to start. exit 101 EOF chmod 0755 "$targetDir/usr/sbin/policy-rc.d" # prevent upstart scripts from running during install/update (but only if the "upstart" package exists) if "$thisDir/debuerreotype-chroot" "$targetDir" apt-get install -qq -s upstart &> /dev/null; then "$thisDir/debuerreotype-chroot" "$targetDir" dpkg-divert --local --rename --add /sbin/initctl > /dev/null cp -a "$targetDir/usr/sbin/policy-rc.d" "$targetDir/sbin/initctl" sed -i 's/^exit.*/exit 0/' "$targetDir/sbin/initctl" fi # force dpkg not to call sync() after package extraction (speeding up installs) if [ -d "$targetDir/etc/dpkg/dpkg.cfg.d" ]; then # --debian-eol lenny and older do not include /etc/dpkg/dpkg.cfg.d cat > "$targetDir/etc/dpkg/dpkg.cfg.d/docker-apt-speedup" <<-'EOF' # For most Docker users, package installs happen during "docker build", which # doesn't survive power loss and gets restarted clean afterwards anyhow, so # this minor tweak gives us a nice speedup (much nicer on spinning disks, # obviously). force-unsafe-io EOF chmod 0644 "$targetDir/etc/dpkg/dpkg.cfg.d/docker-apt-speedup" fi if [ -d "$targetDir/etc/apt/apt.conf.d" ]; then # TODO make some (all?) of these conditional based on the version of APT that added the feature # (perhaps it's finally time for an "apt-version-cmp.sh" helper script to test whether APT is X or newer one version component at a time? "dpkg --compare-versions"!!!) # update "autoremove" configuration to be aggressive about removing suggests deps that weren't manually installed cat > "$targetDir/etc/apt/apt.conf.d/docker-autoremove-suggests" <<-'EOF' # Since Docker users are looking for the smallest possible final images, the # following emerges as a very common pattern: # RUN apt-get update \ # && apt-get install -y \ # && \ # && apt-get purge -y --auto-remove # By default, APT will actually _keep_ packages installed via Recommends or # Depends if another package Suggests them, even and including if the package # that originally caused them to be installed is removed. Setting this to # "false" ensures that APT is appropriately aggressive about removing the # packages it added. # https://aptitude.alioth.debian.org/doc/en/ch02s05s05.html#configApt-AutoRemove-SuggestsImportant Apt::AutoRemove::SuggestsImportant "false"; EOF chmod 0644 "$targetDir/etc/apt/apt.conf.d/docker-autoremove-suggests" "$thisDir/.fix-apt-comments.sh" "$aptVersion" "$targetDir/etc/apt/apt.conf.d/docker-autoremove-suggests" # keep us lean by effectively running "apt-get clean" after every install aptGetClean='"rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true";' cat > "$targetDir/etc/apt/apt.conf.d/docker-clean" <<-EOF # Since for most Docker users, package installs happen in "docker build" steps, # they essentially become individual layers due to the way Docker handles # layering, especially using CoW filesystems. What this means for us is that # the caches that APT keeps end up just wasting space in those layers, making # our layers unnecessarily large (especially since we'll normally never use # these caches again and will instead just "docker build" again and make a brand # new image). # Ideally, these would just be invoking "apt-get clean", but in our testing, # that ended up being cyclic and we got stuck on APT's lock, so we get this fun # creation that's essentially just "apt-get clean". DPkg::Post-Invoke { $aptGetClean }; APT::Update::Post-Invoke { $aptGetClean }; Dir::Cache::pkgcache ""; Dir::Cache::srcpkgcache ""; # Note that we do realize this isn't the ideal way to do this, and are always # open to better suggestions (https://github.com/debuerreotype/debuerreotype/issues). EOF chmod 0644 "$targetDir/etc/apt/apt.conf.d/docker-clean" "$thisDir/.fix-apt-comments.sh" "$aptVersion" "$targetDir/etc/apt/apt.conf.d/docker-clean" cat > "$targetDir/etc/apt/apt.conf.d/docker-gzip-indexes" <<-'EOF' # Since Docker users using "RUN apt-get update && apt-get install -y ..." in # their Dockerfiles don't go delete the lists files afterwards, we want them to # be as small as possible on-disk, so we explicitly request that Apt keep them # compressed on-disk too instead of decompressing them. # For comparison, an "apt-get update" layer without this on a pristine # "debian:wheezy" base image was "29.88 MB", where with this it was only # "8.273 MB". Acquire::GzipIndexes "true"; EOF # https://github.com/debuerreotype/debuerreotype/issues/41 isDebianJessie="$([ -f "$targetDir/etc/os-release" ] && source "$targetDir/etc/os-release" && [ "${ID:-}" = 'debian' ] && [ "${VERSION_ID:-}" = '8' ] && echo '1')" || : if [ -n "$isDebianJessie" ] || [[ "$aptVersion" == 0.* ]] || "$thisDir/debuerreotype-chroot" "$targetDir" dpkg --compare-versions "$aptVersion" '<<' '1.0.9.2~'; then cat >> "$targetDir/etc/apt/apt.conf.d/docker-gzip-indexes" <<-'EOF' # https://salsa.debian.org/apt-team/apt/commit/b0f4b486e6850c5f98520ccf19da71d0ed748ae4; released in src:apt 1.0.9.2, 2014-10-02 # prior to src:apt 1.0.9.2, "Acquire::GzipIndexes" _only_ applied to gzip-compressed list files, so we need to prefer those on older releases Acquire::CompressionTypes::Order:: "gz"; EOF if [ -n "$isDebianJessie" ]; then cat >> "$targetDir/etc/apt/apt.conf.d/docker-gzip-indexes" <<-'EOF' # see also https://github.com/debuerreotype/debuerreotype/issues/41 (details of a bug that's apparently specific to Debian Jessie) EOF fi fi chmod 0644 "$targetDir/etc/apt/apt.conf.d/docker-gzip-indexes" "$thisDir/.fix-apt-comments.sh" "$aptVersion" "$targetDir/etc/apt/apt.conf.d/docker-gzip-indexes" # remove apt-cache translations for faster "apt-get update" cat > "$targetDir/etc/apt/apt.conf.d/docker-no-languages" <<-'EOF' # In Docker, we don't often need the "Translations" files, so we're just wasting # time and space by downloading them, and this inhibits that. For users that do # need them, it's a simple matter to delete this file and "apt-get update". :) Acquire::Languages "none"; EOF chmod 0644 "$targetDir/etc/apt/apt.conf.d/docker-no-languages" "$thisDir/.fix-apt-comments.sh" "$aptVersion" "$targetDir/etc/apt/apt.conf.d/docker-no-languages" fi debuerreotype-0.10/scripts/debuerreotype-slimify000077500000000000000000000054261345767333700223060ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ 'rootfs' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] IFS=$'\n'; set -o noglob slimExcludes=( $(grep -vE '^#|^$' "$thisDir/.slimify-excludes" | sort -u) ) slimIncludes=( $(grep -vE '^#|^$' "$thisDir/.slimify-includes" | sort -u) ) set +o noglob; unset IFS dpkgCfgFile="$targetDir/etc/dpkg/dpkg.cfg.d/docker" mkdir -p "$(dirname "$dpkgCfgFile")" { echo '# This is the "slim" variant of the Debian base image.' echo '# Many files which are normally unnecessary in containers are excluded,' echo '# and this configuration file keeps them that way.' } > "$dpkgCfgFile" findMatchIncludes=() for slimInclude in "${slimIncludes[@]}"; do [ "${#findMatchIncludes[@]}" -eq 0 ] || findMatchIncludes+=( '-o' ) findMatchIncludes+=( -path "$slimInclude" ) done findMatchIncludes=( '(' "${findMatchIncludes[@]}" ')' ) for slimExclude in "${slimExcludes[@]}"; do { echo echo "# dpkg -S '$slimExclude'" if dpkgOutput="$("$thisDir/debuerreotype-chroot" "$targetDir" dpkg -S "$slimExclude" 2>&1)"; then echo "$dpkgOutput" | sed 's/: .*//g; s/, /\n/g' | sort -u | xargs else echo "$dpkgOutput" fi | fold -w 76 -s | sed 's/^/# /' echo "path-exclude $slimExclude" } >> "$dpkgCfgFile" if [[ "$slimExclude" == *'/*' ]]; then if [ -d "$targetDir/$(dirname "$slimExclude")" ]; then # use two passes so that we don't fail trying to remove directories from $slimIncludes # this is our best effort at implementing https://sources.debian.net/src/dpkg/stretch/src/filters.c/#L96-L97 in shell # step 1 -- delete everything that doesn't match "$slimIncludes" and isn't a directory or a symlink "$thisDir/debuerreotype-chroot" "$targetDir" \ find "$(dirname "$slimExclude")" \ -depth -mindepth 1 \ -not \( -type d -o -type l \) \ -not "${findMatchIncludes[@]}" \ -exec rm -f '{}' ';' # step 2 -- repeatedly delete any dangling symlinks and empty directories until there aren't any # (might have a dangling symlink in a directory which then makes it empty, or a symlink to an empty directory) while [ "$( "$thisDir/debuerreotype-chroot" "$targetDir" \ find "$(dirname "$slimExclude")" \ -depth -mindepth 1 \( -empty -o -xtype l \) \ -exec rm -rf '{}' ';' -printf '.' \ | wc -c )" -gt 0 ]; do true; done fi else "$thisDir/debuerreotype-chroot" "$targetDir" rm -f "$slimExclude" fi done { echo for slimInclude in "${slimIncludes[@]}"; do echo "path-include $slimInclude" done } >> "$dpkgCfgFile" chmod 0644 "$dpkgCfgFile" debuerreotype-0.10/scripts/debuerreotype-tar000077500000000000000000000031331345767333700214110ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ --flags 'exclude:' \ --flags 'include-dev' \ -- \ '[--include-dev] ' \ 'rootfs rootfs.tar' eval "$dgetopt" excludes=() includeDev= while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --exclude) excludes+=( "$1" ); shift ;; --include-dev) includeDev=1 ;; --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] targetTar="${1:-}"; shift || eusage 'missing target-tar' [ -n "$targetTar" ] epoch="$(< "$targetDir/debuerreotype-epoch")" [ -n "$epoch" ] aptVersion="$("$thisDir/.apt-version.sh" "$targetDir")" if dpkg --compare-versions "$aptVersion" '>=' '0.8~'; then # if APT is new enough to auto-recreate "partial" directories, let it # (https://salsa.debian.org/apt-team/apt/commit/1cd1c398d18b78f4aa9d882a5de5385f4538e0be) excludes+=( './var/cache/apt/**' './var/lib/apt/lists/**' './var/state/apt/lists/**' ) # (see also the targeted exclusions in ".tar-exclude" that these are overriding) fi "$thisDir/debuerreotype-fixup" "$targetDir" tarArgs=( --create --file "$targetTar" --auto-compress --directory "$targetDir" --exclude-from "$thisDir/.tar-exclude" ) if [ -z "$includeDev" ]; then excludes+=( './dev/**' ) fi for exclude in "${excludes[@]}"; do tarArgs+=( --exclude "$exclude" ) done tarArgs+=( --numeric-owner --transform 's,^./,,' --sort name . ) tar "${tarArgs[@]}" touch --no-dereference --date="@$epoch" "$targetTar" debuerreotype-0.10/scripts/debuerreotype-version000077500000000000000000000004541345767333700223130ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ '' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done _version debuerreotype-0.10/steamos.sh000077500000000000000000000140631345767333700163500ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/scripts/.constants.sh" \ --flags 'no-build' \ -- \ '[--no-build] [suite]' \ 'output' eval "$dgetopt" build=1 while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --no-build) build= ;; # for skipping "docker build" --) break ;; *) eusage "unknown flag '$flag'" ;; esac done outputDir="${1:-}"; shift || eusage 'missing output-dir' suite="${1:-brewmaster}" # http://repo.steampowered.com/steamos/dists/ mkdir -p "$outputDir" outputDir="$(readlink -f "$outputDir")" securityArgs=( --cap-add SYS_ADMIN --cap-drop SETFCAP ) if docker info | grep -q apparmor; then # AppArmor blocks mount :) securityArgs+=( --security-opt apparmor=unconfined ) fi ver="$("$thisDir/scripts/debuerreotype-version")" ver="${ver%% *}" dockerImage="debuerreotype/debuerreotype:$ver" [ -z "$build" ] || docker build -t "$dockerImage" "$thisDir" steamDockerImage="$dockerImage-steamos" [ -z "$build" ] || docker build -t "$steamDockerImage" - <<-EODF FROM $dockerImage # http://repo.steampowered.com/steamos/pool/main/v/valve-archive-keyring/?C=M;O=D RUN wget -O valve.deb 'http://repo.steampowered.com/steamos/pool/main/v/valve-archive-keyring/valve-archive-keyring_0.5+bsos3_all.deb' \\ && apt install -y ./valve.deb \\ && rm valve.deb EODF docker run \ --rm \ "${securityArgs[@]}" \ --tmpfs /tmp:dev,exec,suid,noatime \ -w /tmp \ -e suite="$suite" \ -e TZ='UTC' -e LC_ALL='C' \ "$steamDockerImage" \ bash -Eeuo pipefail -c ' set -x mirror="http://repo.steampowered.com/steamos" dpkgArch="$(dpkg --print-architecture | awk -F- "{ print \$NF }")" exportDir="output" outputDir="$exportDir/steamos/$dpkgArch/$suite" debuerreotypeScriptsDir="$(dirname "$(readlink -f "$(which debuerreotype-init)")")" mkdir -p "$outputDir" wget -O "$outputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg" wget -O "$outputDir/Release" "$mirror/dists/$suite/Release" gpgv \ --keyring /usr/share/keyrings/valve-archive-keyring.gpg \ "$outputDir/Release.gpg" \ "$outputDir/Release" { debuerreotype-init --non-debian \ --debootstrap-script /usr/share/debootstrap/scripts/jessie \ --keyring /usr/share/keyrings/valve-archive-keyring.gpg \ --include valve-archive-keyring \ --exclude debian-archive-keyring \ --no-merged-usr \ rootfs "$suite" "$mirror" echo "deb $mirror $suite main contrib non-free" | tee rootfs/etc/apt/sources.list epoch="$(< rootfs/debuerreotype-epoch)" touch_epoch() { while [ "$#" -gt 0 ]; do local f="$1"; shift touch --no-dereference --date="@$epoch" "$f" done } debuerreotype-minimizing-config rootfs debuerreotype-apt-get rootfs update -qq debuerreotype-apt-get rootfs dist-upgrade -yqq # make a couple copies of rootfs so we can create other variants for variant in slim sbuild; do mkdir "rootfs-$variant" tar -cC rootfs . | tar -xC "rootfs-$variant" done # prefer iproute2 if it exists iproute=iproute2 if ! debuerreotype-chroot rootfs apt-get install -qq -s iproute2 &> /dev/null; then # poor wheezy iproute=iproute fi debuerreotype-apt-get rootfs install -y --no-install-recommends iputils-ping $iproute debuerreotype-slimify rootfs-slim # this should match the list added to the "buildd" variant in debootstrap and the list installed by sbuild # https://anonscm.debian.org/cgit/d-i/debootstrap.git/tree/scripts/sid?id=706a45681c5bba5e062a9b02e19f079cacf2a3e8#n26 # https://anonscm.debian.org/cgit/buildd-tools/sbuild.git/tree/bin/sbuild-createchroot?id=eace3d3e59e48d26eaf069d9b63a6a4c868640e6#n194 fakeroot=fakeroot if [[ "$suite" == alchemist* ]]; then # poor alchemist fakeroot= fi debuerreotype-apt-get rootfs-sbuild install -y --no-install-recommends build-essential $fakeroot create_artifacts() { local targetBase="$1"; shift local rootfs="$1"; shift local suite="$1"; shift local variant="$1"; shift if [ "$variant" != "sbuild" ]; then debuerreotype-tar "$rootfs" "$targetBase.tar.xz" else # sbuild needs "deb-src" entries debuerreotype-chroot "$rootfs" sed -ri -e "/^deb / p; s//deb-src /" /etc/apt/sources.list # APT has odd issues with "Acquire::GzipIndexes=false" + "file://..." sources sometimes # (which are used in sbuild for "--extra-package") # Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) # ... # E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) rm -f "$rootfs/etc/apt/apt.conf.d/docker-gzip-indexes" # TODO figure out the bug and fix it in APT instead /o\ # schroot is picky about "/dev" (which is excluded by default in "debuerreotype-tar") # see https://github.com/debuerreotype/debuerreotype/pull/8#issuecomment-305855521 debuerreotype-tar --include-dev "$rootfs" "$targetBase.tar.xz" fi du -hsx "$targetBase.tar.xz" sha256sum "$targetBase.tar.xz" | cut -d" " -f1 > "$targetBase.tar.xz.sha256" touch_epoch "$targetBase.tar.xz.sha256" debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest" echo "$epoch" > "$targetBase.debuerreotype-epoch" touch_epoch "$targetBase.manifest" "$targetBase.debuerreotype-epoch" for f in debian_version os-release apt/sources.list; do targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")" cp "$rootfs/etc/$f" "$targetFile" touch_epoch "$targetFile" done } for rootfs in rootfs*/; do rootfs="${rootfs%/}" # "rootfs", "rootfs-slim", ... du -hsx "$rootfs" variant="${rootfs#rootfs}" # "", "-slim", ... variant="${variant#-}" # "", "slim", ... variantDir="$outputDir/$variant" mkdir -p "$variantDir" targetBase="$variantDir/rootfs" create_artifacts "$targetBase" "$rootfs" "$suite" "$variant" done } >&2 tar -cC "$exportDir" . ' | tar -xvC "$outputDir" debuerreotype-0.10/ubuntu.sh000077500000000000000000000142721345767333700162210ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/scripts/.constants.sh" \ --flags 'no-build' \ --flags 'arch:' \ -- \ '[--no-build] [--arch=] ' \ 'output xenial --arch arm64 output bionic' eval "$dgetopt" build=1 arch= while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --no-build) build= ;; # for skipping "docker build" --arch) arch="$1"; shift ;; # for adding "--arch" to debuerreotype-init --) break ;; *) eusage "unknown flag '$flag'" ;; esac done outputDir="${1:-}"; shift || eusage 'missing output-dir' suite="${1:-}"; shift || eusage 'missing suite' mkdir -p "$outputDir" outputDir="$(readlink -f "$outputDir")" securityArgs=( --cap-add SYS_ADMIN --cap-drop SETFCAP ) if docker info | grep -q apparmor; then # AppArmor blocks mount :) securityArgs+=( --security-opt apparmor=unconfined ) fi ver="$("$thisDir/scripts/debuerreotype-version")" ver="${ver%% *}" dockerImage="debuerreotype/debuerreotype:$ver" [ -z "$build" ] || docker build -t "$dockerImage" "$thisDir" ubuntuDockerImage="$dockerImage-ubuntu" [ -z "$build" ] || docker build -t "$ubuntuDockerImage" - <<-EODF FROM $dockerImage RUN apt-get update \\ && apt-get install -y --no-install-recommends ubuntu-archive-keyring \\ && rm -rf /var/lib/apt/lists/* EODF docker run \ --rm \ "${securityArgs[@]}" \ -v /tmp \ -w /tmp \ -e suite="$suite" \ -e arch="$arch" \ -e TZ='UTC' -e LC_ALL='C' \ "$ubuntuDockerImage" \ bash -Eeuo pipefail -c ' set -x dpkgArch="${arch:-$(dpkg --print-architecture | awk -F- "{ print \$NF }")}" case "$dpkgArch" in amd64|i386) mirror="http://archive.ubuntu.com/ubuntu" secmirror="http://security.ubuntu.com/ubuntu" ;; *) mirror="http://ports.ubuntu.com/ubuntu-ports" secmirror="$mirror" # no separate security mirror for ports ;; esac exportDir="output" outputDir="$exportDir/ubuntu/$dpkgArch/$suite" debuerreotypeScriptsDir="$(dirname "$(readlink -f "$(which debuerreotype-init)")")" mkdir -p "$outputDir" wget -O "$outputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg" wget -O "$outputDir/Release" "$mirror/dists/$suite/Release" gpgv \ --keyring /usr/share/keyrings/ubuntu-archive-keyring.gpg \ "$outputDir/Release.gpg" \ "$outputDir/Release" { debuerreotype-init --non-debian \ --arch="$dpkgArch" \ --keyring /usr/share/keyrings/ubuntu-archive-keyring.gpg \ --no-merged-usr \ rootfs "$suite" "$mirror" # TODO setup proper sources.list for Ubuntu # deb http://archive.ubuntu.com/ubuntu xenial main restricted universe multiverse # deb http://archive.ubuntu.com/ubuntu xenial-updates main restricted universe multiverse # deb http://archive.ubuntu.com/ubuntu xenial-backports main restricted universe multiverse # deb http://security.ubuntu.com/ubuntu xenial-security main restricted universe multiverse epoch="$(< rootfs/debuerreotype-epoch)" touch_epoch() { while [ "$#" -gt 0 ]; do local f="$1"; shift touch --no-dereference --date="@$epoch" "$f" done } debuerreotype-minimizing-config rootfs debuerreotype-apt-get rootfs update -qq debuerreotype-apt-get rootfs dist-upgrade -yqq # make a couple copies of rootfs so we can create other variants for variant in slim sbuild; do mkdir "rootfs-$variant" tar -cC rootfs . | tar -xC "rootfs-$variant" done debuerreotype-apt-get rootfs install -y --no-install-recommends iproute2 iputils-ping debuerreotype-slimify rootfs-slim # this should match the list added to the "buildd" variant in debootstrap and the list installed by sbuild # https://anonscm.debian.org/cgit/d-i/debootstrap.git/tree/scripts/sid?id=706a45681c5bba5e062a9b02e19f079cacf2a3e8#n26 # https://anonscm.debian.org/cgit/buildd-tools/sbuild.git/tree/bin/sbuild-createchroot?id=eace3d3e59e48d26eaf069d9b63a6a4c868640e6#n194 debuerreotype-apt-get rootfs-sbuild install -y --no-install-recommends build-essential fakeroot create_artifacts() { local targetBase="$1"; shift local rootfs="$1"; shift local suite="$1"; shift local variant="$1"; shift if [ "$variant" != "sbuild" ]; then debuerreotype-tar "$rootfs" "$targetBase.tar.xz" else # sbuild needs "deb-src" entries debuerreotype-chroot "$rootfs" sed -ri -e "/^deb / p; s//deb-src /" /etc/apt/sources.list # APT has odd issues with "Acquire::GzipIndexes=false" + "file://..." sources sometimes # (which are used in sbuild for "--extra-package") # Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) # ... # E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) rm -f "$rootfs/etc/apt/apt.conf.d/docker-gzip-indexes" # TODO figure out the bug and fix it in APT instead /o\ # schroot is picky about "/dev" (which is excluded by default in "debuerreotype-tar") # see https://github.com/debuerreotype/debuerreotype/pull/8#issuecomment-305855521 debuerreotype-tar --include-dev "$rootfs" "$targetBase.tar.xz" fi du -hsx "$targetBase.tar.xz" sha256sum "$targetBase.tar.xz" | cut -d" " -f1 > "$targetBase.tar.xz.sha256" touch_epoch "$targetBase.tar.xz.sha256" debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest" echo "$epoch" > "$targetBase.debuerreotype-epoch" touch_epoch "$targetBase.manifest" "$targetBase.debuerreotype-epoch" for f in debian_version os-release apt/sources.list; do targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")" cp "$rootfs/etc/$f" "$targetFile" touch_epoch "$targetFile" done } for rootfs in rootfs*/; do rootfs="${rootfs%/}" # "rootfs", "rootfs-slim", ... du -hsx "$rootfs" variant="${rootfs#rootfs}" # "", "-slim", ... variant="${variant#-}" # "", "slim", ... variantDir="$outputDir/$variant" mkdir -p "$variantDir" targetBase="$variantDir/rootfs" create_artifacts "$targetBase" "$rootfs" "$suite" "$variant" done } >&2 tar -cC "$exportDir" . ' | tar -xvC "$outputDir"