pax_global_header00006660000000000000000000000064131542474020014514gustar00rootroot0000000000000052 comment=ba2ce13905d51a16ef60cb0f32214e230e75329b debuerreotype-0.4/000077500000000000000000000000001315424740200142355ustar00rootroot00000000000000debuerreotype-0.4/.dockerignore000066400000000000000000000000151315424740200167050ustar00rootroot00000000000000** !scripts/ debuerreotype-0.4/.travis.sh000077500000000000000000000007541315424740200161700ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail epoch="$(TZ=UTC date --date "$TIMESTAMP" +%s)" serial="$(TZ=UTC date --date "@$epoch" +%Y%m%d)" buildArgs=() if [ -n "${CODENAME:-}" ]; then buildArgs+=( '--codename-copy' ) fi buildArgs+=( travis "$SUITE" "@$epoch" ) checkFile="travis/$serial/amd64/${CODENAME:-$SUITE}/rootfs.tar.xz" set -x ./scripts/debuerreotype-version ./build.sh "${buildArgs[@]}" real="$(sha256sum "$checkFile" | cut -d' ' -f1)" [ -z "$SHA256" ] || [ "$SHA256" = "$real" ] debuerreotype-0.4/.travis.yml000066400000000000000000000023321315424740200163460ustar00rootroot00000000000000language: bash services: docker env: - SUITE=jessie CODENAME= TIMESTAMP=2017-01-01T00:00:00Z SHA256=881da2a3f289dd665a44e3b0f87501a39a98584a587746963cf761ef7b612c02 - SUITE=stable CODENAME=jessie TIMESTAMP=2017-01-01T00:00:00Z SHA256=881da2a3f289dd665a44e3b0f87501a39a98584a587746963cf761ef7b612c02 - SUITE=stretch CODENAME= TIMESTAMP=2017-01-01T00:00:00Z SHA256=139ed970d52ef950c223f9ab325657eb93d0a93c7d6e2fc697fe7510e61760fa - SUITE=testing CODENAME=stretch TIMESTAMP=2017-01-01T00:00:00Z SHA256=139ed970d52ef950c223f9ab325657eb93d0a93c7d6e2fc697fe7510e61760fa - SUITE=sid CODENAME= TIMESTAMP=2017-01-01T00:00:00Z SHA256=b75b4496deb4d6cee32245e4125e7ef948b09afbeb1ef3b9669e56daf3e822a7 - SUITE=unstable CODENAME=sid TIMESTAMP=2017-01-01T00:00:00Z SHA256=b75b4496deb4d6cee32245e4125e7ef948b09afbeb1ef3b9669e56daf3e822a7 - SUITE=wheezy CODENAME= TIMESTAMP=2017-01-01T00:00:00Z SHA256=b4f7069aba089e014590e51437ab03dc1b827372711fe4860866bfe9d920e956 - SUITE=oldstable CODENAME=wheezy TIMESTAMP=2017-01-01T00:00:00Z SHA256=b4f7069aba089e014590e51437ab03dc1b827372711fe4860866bfe9d920e956 script: - travis_retry ./.travis.sh after_script: - docker images debuerreotype-0.4/Dockerfile000066400000000000000000000026661315424740200162410ustar00rootroot00000000000000# docker run --cap-add SYS_ADMIN --tmpfs /tmp:dev,exec,suid,noatime ... # bootstrapping a new architecture? # ./scripts/debuerreotype-init /tmp/docker-rootfs stretch now # ./scripts/debuerreotype-minimizing-config /tmp/docker-rootfs # ./scripts/debuerreotype-gen-sources-list /tmp/docker-rootfs stretch http://deb.debian.org/debian http://security.debian.org # ./scripts/debuerreotype-tar /tmp/docker-rootfs - | docker import - debian:stretch-slim # alternate: # debootstrap --variant=minbase stretch /tmp/docker-rootfs # tar -cC /tmp/docker-rootfs . | docker import - debian:stretch-slim # (or your own favorite set of "debootstrap" commands to create a base image for building this one FROM) FROM debian:stretch-slim RUN apt-get update && apt-get install -y --no-install-recommends \ debootstrap \ wget ca-certificates \ xz-utils \ && rm -rf /var/lib/apt/lists/* COPY scripts /opt/debuerreotype/scripts RUN set -ex; \ cd /opt/debuerreotype/scripts; \ for f in debuerreotype-*; do \ ln -svL "$PWD/$f" "/usr/local/bin/$f"; \ done WORKDIR /tmp # a few example md5sum values for amd64: # debuerreotype-init test-stretch stretch 2017-05-08T00:00:00Z # debuerreotype-tar test-stretch test-stretch.tar # md5sum test-stretch.tar # 6f965e84837215ac0aa375e3391392db # debuerreotype-init test-jessie jessie 2017-05-08T00:00:00Z # debuerreotype-tar test-jessie test-jessie.tar # md5sum test-jessie.tar # 93ad9886b0e0da17aae584d3a0236d0c debuerreotype-0.4/LICENSE000066400000000000000000000020601315424740200152400ustar00rootroot00000000000000Copyright 2017 Tianon Gravi Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. debuerreotype-0.4/README.md000066400000000000000000000150461315424740200155220ustar00rootroot00000000000000# Debuerreotype [![Build Status](https://travis-ci.org/debuerreotype/debuerreotype.svg?branch=master)](https://travis-ci.org/debuerreotype/debuerreotype/branches) Reproducible, [snapshot](http://snapshot.debian.org)-based Debian rootfs builds (especially for Docker). This is based on [lamby](https://github.com/lamby)'s work for reproducible `debootstrap`: - https://github.com/lamby/debootstrap/commit/66b15380814aa62ca4b5807270ac57a3c8a0558d - https://wiki.debian.org/ReproducibleInstalls ### "Debuerreotype"? The name is an attempt at riffing off the photography basis of the word "snapshot". The [daguerreotype](https://en.wikipedia.org/wiki/Daguerreotype) process was an early method for taking photographs, and this is a method for taking "photographs" of Debian at a given point in time. ## Why? The goal is to create an auditable, reproducible process for creating rootfs tarballs (especially for use in Docker) of Debian releases, based on point-in-time snapshots from [snapshot.debian.org](http://snapshot.debian.org). However, as noted below, the only strictly Docker-specific script is `debuerreotype-minimizing-config`, which applies many configuration tweaks which are useful for Docker users and may or may not be useful outside of that context. ## Usage The usage of the scripts here center around a "rootfs" directory, which is both the working directory for building the target rootfs, and contains the `debuerreotype-epoch` file, which records our snapshot.debian.org epoch value (so we can adjust timestamps using it, as it is the basis for our reproducibility). Available scripts: | *script* | *purpose* | | --- | --- | | `debuerreotype-init` | create the initial "rootfs", given a suite and a timestamp (in some format `date(1)` can parse); `sources.list` will be pointing at snapshot.debian.org | | `debuerreotype-chroot` | run a command in the given "rootfs" (using `unshare` to mount `/dev`, `/proc`, and `/sys` from the parent environment in a simple, safe way) | | `debuerreotype-apt-get` | run `apt-get` via `debuerreotype-chroot`, including `-o Acquire::Check-Valid-Until=false` to account for older snapshots with (now) invalid `Valid-Until` values | | `debuerreotype-minimizing-config` | apply configuration tweaks to make the rootfs minimal and keep it minimal (especially targeted at Docker images, with comments explicitly describing Docker use cases) | | `debuerreotype-slimify` | remove files such as documentation to create an even smaller rootfs (used for creating `slim` variants of the Docker images, for example) | | `debuerreotype-gen-sources-list` | generate an appropriate `sources.list` in the rootfs given a suite, mirror, and secmirror (especially for updating `sources.list` to point at deb.debian.org before generating outputs) | | `debuerreotype-fixup` | invoked by `debuerreotype-tar` to fixup timestamps and remove known-bad log files for determinism | | `debuerreotype-tar` | deterministically create a tar file of the rootfs | | `debuerreotype-version` | print out the version of the current `debuerreotype` installation | A simple `Dockerfile` is provided for using these scripts in a simple deterministic environment based on Docker, but given a recent enough version of `debootstrap`, they should run fine outside Docker as well (and their deterministic properties have been verified on at least a Gentoo host in addition to the provided Debian-based Docker environment). The provided `Dockerfile` also includes comments with hints for bootstrapping the environment on a new architecture (which then presumably doesn't have a `debian` Docker base image yet). Full example: (see [`build.sh`](build.sh) for this in practice) ```console $ debuerreotype-init rootfs stretch 2017-01-01T00:00:00Z I: Retrieving InRelease I: Checking Release signature I: Valid Release signature (key id 126C0D24BD8A2942CC7DF8AC7638D0442B90D010) ... I: Checking component main on http://snapshot.debian.org/archive/debian/20170101T000000Z... ... I: Base system installed successfully. $ cat rootfs/debuerreotype-epoch 1483228800 $ debuerreotype-minimizing-config rootfs $ debuerreotype-apt-get rootfs update -qq $ debuerreotype-apt-get rootfs dist-upgrade -yqq $ debuerreotype-apt-get rootfs install -yqq --no-install-recommends inetutils-ping iproute2 debconf: delaying package configuration, since apt-utils is not installed Selecting previously unselected package libelf1:amd64. (Reading database ... 6299 files and directories currently installed.) Preparing to unpack .../0-libelf1_0.166-2.2_amd64.deb ... Unpacking libelf1:amd64 (0.166-2.2) ... Selecting previously unselected package libmnl0:amd64. Preparing to unpack .../1-libmnl0_1.0.4-2_amd64.deb ... Unpacking libmnl0:amd64 (1.0.4-2) ... Selecting previously unselected package iproute2. Preparing to unpack .../2-iproute2_4.9.0-1_amd64.deb ... Unpacking iproute2 (4.9.0-1) ... Selecting previously unselected package netbase. Preparing to unpack .../3-netbase_5.3_all.deb ... Unpacking netbase (5.3) ... Selecting previously unselected package inetutils-ping. Preparing to unpack .../4-inetutils-ping_2%3a1.9.4-2+b1_amd64.deb ... Unpacking inetutils-ping (2:1.9.4-2+b1) ... Setting up libelf1:amd64 (0.166-2.2) ... Processing triggers for libc-bin (2.24-8) ... Setting up libmnl0:amd64 (1.0.4-2) ... Setting up netbase (5.3) ... Setting up inetutils-ping (2:1.9.4-2+b1) ... Setting up iproute2 (4.9.0-1) ... Processing triggers for libc-bin (2.24-8) ... $ debuerreotype-gen-sources-list rootfs stretch http://deb.debian.org/debian http://security.debian.org $ debuerreotype-tar rootfs - | sha256sum 0542bec04135ed60ed5763f0bcf90381d4e5e33786d57aba5aa4b0fc4e43478a - $ # try it! you should get that same sha256sum value! ``` ## How much have you verified this? Well, I ran the scripts across seven explicit architectures (`amd64`, `arm64`, `armel`, `armhf`, `i386`, `ppc64el`, `s390x`) and eight explicit suites (`oldstable`, `stable`, `testing`, `unstable`, `wheezy`, `jessie`, `stretch`, `sid`) for a timestamp of `2017-05-16T00:00:00Z` (where supported, since `wheezy`/`oldstable` didn't or no longer currently supports some of those architectures), and there were no modifications to any of the tarballs after several runs across several days. Additionally, Travis runs with a fixed timestamp value across several suites to verify that their checksums are reproducible, as expected. From time to time, comments in the files generated by `debuerreotype-minimizing-config` might change (for example), which would obviously result in a different checksum, but a simple [`diffoscope`](https://diffoscope.org/) should be sufficient to verify that the change is benign. debuerreotype-0.4/VERSION000066400000000000000000000000041315424740200152770ustar00rootroot000000000000000.4 debuerreotype-0.4/build-all.sh000077500000000000000000000034361315424740200164470ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail suites=( unstable testing stable oldstable oldoldstable # just in case (will no-op with "not supported on 'arch'" unless it exists) oldoldoldstable ) thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/scripts/.constants.sh" \ --flags 'no-build' \ -- \ '[--no-build] ' \ 'output 2017-05-08T00:00:00Z' eval "$dgetopt" build=1 while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --no-build) build= ;; # for skipping "docker build" --) break ;; *) eusage "unknown flag '$flag'" ;; esac done outputDir="${1:-}"; shift || eusage 'missing output-dir' timestamp="${1:-}"; shift || eusage 'missing timestamp' mkdir -p "$outputDir" outputDir="$(readlink -f "$outputDir")" ver="$("$thisDir/scripts/debuerreotype-version")" ver="${ver%% *}" dockerImage="debuerreotype/debuerreotype:$ver" [ -z "$build" ] || docker build -t "$dockerImage" "$thisDir" mirror="$("$thisDir/scripts/.snapshot-url.sh" "$timestamp")" secmirror="$("$thisDir/scripts/.snapshot-url.sh" "$timestamp" 'debian-security')" dpkgArch="$(docker run --rm "$dockerImage" dpkg --print-architecture)" echo echo "-- BUILDING TARBALLS FOR '$dpkgArch' FROM '$mirror/' --" echo for suite in "${suites[@]}"; do doSkip= case "$suite" in testing|unstable) ;; *) if ! wget --quiet --spider "$secmirror/dists/$suite/updates/main/binary-$dpkgArch/Packages.gz"; then doSkip=1 fi ;; esac if ! wget --quiet --spider "$mirror/dists/$suite/main/binary-$dpkgArch/Packages.gz"; then doSkip=1 fi if [ -n "$doSkip" ]; then echo >&2 echo >&2 "warning: '$suite' not supported on '$dpkgArch' (at '$timestamp'); skipping" echo >&2 continue fi "$thisDir/build.sh" --no-build --codename-copy "$outputDir" "$suite" "$timestamp" done debuerreotype-0.4/build.sh000077500000000000000000000177111315424740200157020ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/scripts/.constants.sh" \ --flags 'no-build,codename-copy' \ -- \ '[--no-build] [--codename-copy] ' \ 'output stretch 2017-05-08T00:00:00Z --codename-copy output stable 2017-05-08T00:00:00Z' eval "$dgetopt" build=1 codenameCopy= while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --no-build) build= ;; # for skipping "docker build" --codename-copy) codenameCopy=1 ;; # for copying a "stable.tar.xz" to "stretch.tar.xz" with updated sources.list (saves a lot of extra building work) --) break ;; *) eusage "unknown flag '$flag'" ;; esac done outputDir="${1:-}"; shift || eusage 'missing output-dir' suite="${1:-}"; shift || eusage 'missing suite' timestamp="${1:-}"; shift || eusage 'missing timestamp' mkdir -p "$outputDir" outputDir="$(readlink -f "$outputDir")" securityArgs=( --cap-add SYS_ADMIN ) if docker info | grep -q apparmor; then # AppArmor blocks mount :) securityArgs+=( --security-opt apparmor=unconfined ) fi ver="$("$thisDir/scripts/debuerreotype-version")" ver="${ver%% *}" dockerImage="debuerreotype/debuerreotype:$ver" [ -z "$build" ] || docker build -t "$dockerImage" "$thisDir" docker run \ --rm \ "${securityArgs[@]}" \ --tmpfs /tmp:dev,exec,suid,noatime \ -w /tmp \ -e suite="$suite" \ -e timestamp="$timestamp" \ -e codenameCopy="$codenameCopy" \ -e TZ='UTC' -e LC_ALL='C' \ "$dockerImage" \ bash -Eeuo pipefail -c ' set -x epoch="$(date --date "$timestamp" +%s)" serial="$(date --date "@$epoch" +%Y%m%d)" dpkgArch="$(dpkg --print-architecture)" exportDir="output" outputDir="$exportDir/$serial/$dpkgArch/$suite" touch_epoch() { while [ "$#" -gt 0 ]; do local f="$1"; shift touch --no-dereference --date="@$epoch" "$f" done } debuerreotypeScriptsDir="$(dirname "$(readlink -f "$(which debuerreotype-init)")")" for archive in "" security; do snapshotUrl="$("$debuerreotypeScriptsDir/.snapshot-url.sh" "@$epoch" "${archive:+debian-${archive}}")" snapshotUrlFile="$exportDir/$serial/$dpkgArch/snapshot-url${archive:+-${archive}}" mkdir -p "$(dirname "$snapshotUrlFile")" echo "$snapshotUrl" > "$snapshotUrlFile" touch_epoch "$snapshotUrlFile" done snapshotUrl="$(< "$exportDir/$serial/$dpkgArch/snapshot-url")" mkdir -p "$outputDir" wget -O "$outputDir/Release.gpg" "$snapshotUrl/dists/$suite/Release.gpg" wget -O "$outputDir/Release" "$snapshotUrl/dists/$suite/Release" gpgv \ --keyring /usr/share/keyrings/debian-archive-keyring.gpg \ --keyring /usr/share/keyrings/debian-archive-removed-keys.gpg \ "$outputDir/Release.gpg" \ "$outputDir/Release" codename="$(awk -F ": " "\$1 == \"Codename\" { print \$2; exit }" "$outputDir/Release")" if [ -n "$codenameCopy" ] && [ "$codename" = "$suite" ]; then # if codename already is the same as suite, then making a copy does not make any sense codenameCopy= fi if [ -n "$codenameCopy" ] && [ -z "$codename" ]; then echo >&2 "error: --codename-copy specified but we failed to get a Codename for $suite" exit 1 fi { initArgs=( --debian ) releaseSuite="$(awk -F ": " "\$1 == \"Suite\" { print \$2; exit }" "$outputDir/Release")" case "$suite" in # see https://bugs.debian.org/src:usrmerge for why merged-usr should not be in stable yet (mostly "dpkg" related bugs) *oldstable|stable) initArgs+=( --no-merged-usr ) ;; esac debuerreotype-init "${initArgs[@]}" rootfs "$suite" "@$epoch" debuerreotype-minimizing-config rootfs debuerreotype-apt-get rootfs update -qq debuerreotype-apt-get rootfs dist-upgrade -yqq # make a couple copies of rootfs so we can create other variants for variant in slim sbuild; do mkdir "rootfs-$variant" tar -cC rootfs . | tar -xC "rootfs-$variant" done # prefer iproute2 if it exists iproute=iproute2 if ! debuerreotype-chroot rootfs apt-cache show iproute2 > /dev/null; then # poor wheezy iproute=iproute fi debuerreotype-apt-get rootfs install -y --no-install-recommends inetutils-ping $iproute debuerreotype-slimify rootfs-slim # this should match the list added to the "buildd" variant in debootstrap and the list installed by sbuild # https://anonscm.debian.org/cgit/d-i/debootstrap.git/tree/scripts/sid?id=706a45681c5bba5e062a9b02e19f079cacf2a3e8#n26 # https://anonscm.debian.org/cgit/buildd-tools/sbuild.git/tree/bin/sbuild-createchroot?id=eace3d3e59e48d26eaf069d9b63a6a4c868640e6#n194 debuerreotype-apt-get rootfs-sbuild install -y --no-install-recommends build-essential fakeroot create_artifacts() { local targetBase="$1"; shift local rootfs="$1"; shift local suite="$1"; shift local variant="$1"; shift # make a copy of the snapshot-facing sources.list file before we overwrite it cp "$rootfs/etc/apt/sources.list" "$targetBase.sources-list-snapshot" touch_epoch "$targetBase.sources-list-snapshot" if [ "$variant" != "sbuild" ]; then debuerreotype-gen-sources-list "$rootfs" "$suite" http://deb.debian.org/debian http://security.debian.org debuerreotype-tar "$rootfs" "$targetBase.tar.xz" else # sbuild needs "deb-src" entries debuerreotype-gen-sources-list --deb-src "$rootfs" "$suite" http://deb.debian.org/debian http://security.debian.org # APT has odd issues with "Acquire::GzipIndexes=false" + "file://..." sources sometimes # (which are used in sbuild for "--extra-package") # Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) # ... # E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) rm -f "$rootfs/etc/apt/apt.conf.d/docker-gzip-indexes" # TODO figure out the bug and fix it in APT instead /o\ # schroot is picky about "/dev" (which is excluded by default in "debuerreotype-tar") # see https://github.com/debuerreotype/debuerreotype/pull/8#issuecomment-305855521 debuerreotype-tar --include-dev "$rootfs" "$targetBase.tar.xz" fi du -hsx "$targetBase.tar.xz" sha256sum "$targetBase.tar.xz" | cut -d" " -f1 > "$targetBase.tar.xz.sha256" touch_epoch "$targetBase.tar.xz.sha256" debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest" echo "$epoch" > "$targetBase.debuerreotype-epoch" touch_epoch "$targetBase.manifest" "$targetBase.debuerreotype-epoch" for f in debian_version os-release apt/sources.list; do targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")" cp "$rootfs/etc/$f" "$targetFile" touch_epoch "$targetFile" done } for rootfs in rootfs*/; do rootfs="${rootfs%/}" # "rootfs", "rootfs-slim", ... du -hsx "$rootfs" variant="${rootfs#rootfs}" # "", "-slim", ... variant="${variant#-}" # "", "slim", ... variantDir="$outputDir/$variant" mkdir -p "$variantDir" targetBase="$variantDir/rootfs" create_artifacts "$targetBase" "$rootfs" "$suite" "$variant" done if [ -n "$codenameCopy" ]; then codenameDir="$exportDir/$serial/$dpkgArch/$codename" mkdir -p "$codenameDir" tar -cC "$outputDir" --exclude="**/rootfs.*" . | tar -xC "$codenameDir" for rootfs in rootfs*/; do rootfs="${rootfs%/}" # "rootfs", "rootfs-slim", ... variant="${rootfs#rootfs}" # "", "-slim", ... variant="${variant#-}" # "", "slim", ... variantDir="$codenameDir/$variant" targetBase="$variantDir/rootfs" # point sources.list back at snapshot.debian.org temporarily (but this time pointing at $codename instead of $suite) debuerreotype-gen-sources-list "$rootfs" "$codename" "$(< "$exportDir/$serial/$dpkgArch/snapshot-url")" "$(< "$exportDir/$serial/$dpkgArch/snapshot-url-security")" create_artifacts "$targetBase" "$rootfs" "$codename" "$variant" done fi } >&2 tar -cC "$exportDir" . ' | tar -xvC "$outputDir" debuerreotype-0.4/raspbian.sh000077500000000000000000000131321315424740200163730ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/scripts/.constants.sh" \ --flags 'no-build' \ -- \ '[--no-build] ' \ 'output stretch' eval "$dgetopt" build=1 while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --no-build) build= ;; # for skipping "docker build" --) break ;; *) eusage "unknown flag '$flag'" ;; esac done outputDir="${1:-}"; shift || eusage 'missing output-dir' suite="${1:-}"; shift || eusage 'missing suite' mkdir -p "$outputDir" outputDir="$(readlink -f "$outputDir")" securityArgs=( --cap-add SYS_ADMIN ) if docker info | grep -q apparmor; then # AppArmor blocks mount :) securityArgs+=( --security-opt apparmor=unconfined ) fi ver="$("$thisDir/scripts/debuerreotype-version")" ver="${ver%% *}" dockerImage="debuerreotype/debuerreotype:$ver" [ -z "$build" ] || docker build -t "$dockerImage" "$thisDir" raspbianDockerImage="$dockerImage-raspbian" [ -z "$build" ] || docker build -t "$raspbianDockerImage" - <<-EODF FROM $dockerImage RUN wget -O raspbian.deb 'https://archive.raspbian.org/raspbian/pool/main/r/raspbian-archive-keyring/raspbian-archive-keyring-udeb_20120528.2_all.udeb' \\ && apt install -y ./raspbian.deb \\ && rm raspbian.deb EODF docker run \ --rm \ "${securityArgs[@]}" \ -v /tmp \ -w /tmp \ -e suite="$suite" \ -e TZ='UTC' -e LC_ALL='C' \ "$raspbianDockerImage" \ bash -Eeuo pipefail -c ' set -x mirror="http://archive.raspbian.org/raspbian" dpkgArch="$(dpkg --print-architecture)" exportDir="output" outputDir="$exportDir/raspbian/$dpkgArch/$suite" debuerreotypeScriptsDir="$(dirname "$(readlink -f "$(which debuerreotype-init)")")" mkdir -p "$outputDir" wget -O "$outputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg" wget -O "$outputDir/Release" "$mirror/dists/$suite/Release" gpgv \ --keyring /usr/share/keyrings/raspbian-archive-keyring.gpg \ "$outputDir/Release.gpg" \ "$outputDir/Release" { debuerreotype-init --non-debian \ --arch armhf \ --keyring /usr/share/keyrings/raspbian-archive-keyring.gpg \ rootfs "$suite" "$mirror" epoch="$(< rootfs/debuerreotype-epoch)" touch_epoch() { while [ "$#" -gt 0 ]; do local f="$1"; shift touch --no-dereference --date="@$epoch" "$f" done } debuerreotype-minimizing-config rootfs debuerreotype-apt-get rootfs update -qq debuerreotype-apt-get rootfs dist-upgrade -yqq # make a couple copies of rootfs so we can create other variants for variant in slim sbuild; do mkdir "rootfs-$variant" tar -cC rootfs . | tar -xC "rootfs-$variant" done # prefer iproute2 if it exists iproute=iproute2 if ! debuerreotype-chroot rootfs apt-cache show iproute2 > /dev/null; then # poor wheezy iproute=iproute fi debuerreotype-apt-get rootfs install -y --no-install-recommends inetutils-ping $iproute debuerreotype-slimify rootfs-slim # this should match the list added to the "buildd" variant in debootstrap and the list installed by sbuild # https://anonscm.debian.org/cgit/d-i/debootstrap.git/tree/scripts/sid?id=706a45681c5bba5e062a9b02e19f079cacf2a3e8#n26 # https://anonscm.debian.org/cgit/buildd-tools/sbuild.git/tree/bin/sbuild-createchroot?id=eace3d3e59e48d26eaf069d9b63a6a4c868640e6#n194 debuerreotype-apt-get rootfs-sbuild install -y --no-install-recommends build-essential fakeroot create_artifacts() { local targetBase="$1"; shift local rootfs="$1"; shift local suite="$1"; shift local variant="$1"; shift if [ "$variant" != "sbuild" ]; then debuerreotype-tar "$rootfs" "$targetBase.tar.xz" else # sbuild needs "deb-src" entries debuerreotype-chroot "$rootfs" sed -ri -e "/^deb / p; s//deb-src /" /etc/apt/sources.list # APT has odd issues with "Acquire::GzipIndexes=false" + "file://..." sources sometimes # (which are used in sbuild for "--extra-package") # Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) # ... # E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) rm -f "$rootfs/etc/apt/apt.conf.d/docker-gzip-indexes" # TODO figure out the bug and fix it in APT instead /o\ # schroot is picky about "/dev" (which is excluded by default in "debuerreotype-tar") # see https://github.com/debuerreotype/debuerreotype/pull/8#issuecomment-305855521 debuerreotype-tar --include-dev "$rootfs" "$targetBase.tar.xz" fi du -hsx "$targetBase.tar.xz" sha256sum "$targetBase.tar.xz" | cut -d" " -f1 > "$targetBase.tar.xz.sha256" touch_epoch "$targetBase.tar.xz.sha256" debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest" echo "$epoch" > "$targetBase.debuerreotype-epoch" touch_epoch "$targetBase.manifest" "$targetBase.debuerreotype-epoch" for f in debian_version os-release apt/sources.list; do targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")" cp "$rootfs/etc/$f" "$targetFile" touch_epoch "$targetFile" done } for rootfs in rootfs*/; do rootfs="${rootfs%/}" # "rootfs", "rootfs-slim", ... du -hsx "$rootfs" variant="${rootfs#rootfs}" # "", "-slim", ... variant="${variant#-}" # "", "slim", ... variantDir="$outputDir/$variant" mkdir -p "$variantDir" targetBase="$variantDir/rootfs" create_artifacts "$targetBase" "$rootfs" "$suite" "$variant" done } >&2 tar -cC "$exportDir" . ' | tar -xvC "$outputDir" debuerreotype-0.4/scripts/000077500000000000000000000000001315424740200157245ustar00rootroot00000000000000debuerreotype-0.4/scripts/.constants.sh000066400000000000000000000036171315424740200203610ustar00rootroot00000000000000#!/usr/bin/env bash # constants of the universe export TZ='UTC' LC_ALL='C' umask 0002 scriptsDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" self="$(basename "$0")" options="$(getopt -n "$BASH_SOURCE" -o '+' --long 'flags:,flags-short:' -- "$@")" dFlags='help,version' dFlagsShort='h?' usageStr= __cgetopt() { eval "set -- $options" # in a function since otherwise "set" will overwrite the parent script's positional args too unset options while true; do local flag="$1"; shift case "$flag" in --flags) dFlags="${dFlags:+$dFlags,}$1"; shift ;; --flags-short) dFlagsShort="${dFlagsShort}$1"; shift ;; --) break ;; *) echo >&2 "error: unexpected $BASH_SOURCE flag '$flag'"; exit 1 ;; esac done while [ "$#" -gt 0 ]; do local IFS=$'\n' local usagePrefix='usage:' usageLine= for usageLine in $1; do usageStr+="$usagePrefix $self${usageLine:+ $usageLine}"$'\n' usagePrefix=' ' done usagePrefix=' ie:' for usageLine in $2; do usageStr+="$usagePrefix $self${usageLine:+ $usageLine}"$'\n' usagePrefix=' ' done usageStr+=$'\n' shift 2 done } __cgetopt _version() { local v if [ -r "$scriptsDir/../VERSION" ]; then v="$(< "$scriptsDir/../VERSION")" else v='unknown' fi if [ -d "$scriptsDir/../.git" ] && command -v git > /dev/null; then local commit="$(git -C "$scriptsDir" rev-parse --short 'HEAD^{commit}')" v="$v commit $commit" fi echo "$v" } usage() { echo -n "$usageStr" local v="$(_version)" echo "debuerreotype version $v" } eusage() { if [ "$#" -gt 0 ]; then echo >&2 "error: $*"$'\n' fi usage >&2 exit 1 } _dgetopt() { getopt -n "$self" \ -o "+$dFlagsShort" \ --long "$dFlags" \ -- "$@" \ || eusage 'getopt failed' } dgetopt='options="$(_dgetopt "$@")"; eval "set -- $options"; unset options' dgetopt-case() { local flag="$1"; shift case "$flag" in -h|'-?'|--help) usage; exit 0 ;; --version) _version; exit 0 ;; esac } debuerreotype-0.4/scripts/.slimify-excludes000066400000000000000000000005521315424740200212150ustar00rootroot00000000000000# This file contains the list of files/directories which will be removed for "slim" image variants. # https://github.com/tianon/docker-brew-debian/issues/48 # https://wiki.ubuntu.com/ReducingDiskFootprint#Drop_unnecessary_files /usr/share/doc/* /usr/share/groff/* /usr/share/info/* /usr/share/linda/* /usr/share/lintian/* /usr/share/locale/* /usr/share/man/* debuerreotype-0.4/scripts/.snapshot-url.sh000077500000000000000000000005731315424740200210050ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ ' [archive]' \ '2017-05-08T00:00:00Z debian-security' timestamp="${1:-}"; shift || eusage 'missing timestamp' archive="${1:-debian}" t="$(date --date "$timestamp" '+%Y%m%dT%H%M%SZ')" echo "http://snapshot.debian.org/archive/$archive/$t" debuerreotype-0.4/scripts/.tar-exclude000066400000000000000000000020471315424740200201450ustar00rootroot00000000000000# the file we store the "epoch" of a given rootfs in ./debuerreotype-epoch # "/dev" is a special case in "debuerreotype-tar" #./dev/** ./proc/** ./sys/** ./var/cache/apt/** ./var/lib/apt/lists/** # ends up with host-kernel info ./etc/apt/apt.conf.d/01autoremove-kernels # useful data in these, but includes timestamps too ./var/log/apt/history.log ./var/log/apt/term.log # wheezy-only file which contains host-kernel info ./run/motd.dynamic # wheezy-only file which APT doesn't even use (and has timestamp embedded in binary data) # see https://github.com/debuerreotype/debuerreotype/issues/7 ./etc/apt/trustdb.gpg # Debian creates this file reproducibly, but Ubuntu does not # (according to "man 1 journalctl", this is automatically recreated by "journalctl --update-catalog") # Tails also removes this file to achieve reproducibility (https://labs.riseup.net/code/projects/tails/repository/revisions/b1e05c8aac12fc79293f6a220b40a538d4f38c51/diff/config/chroot_local-hooks/99-zzzzzz_reproducible-builds-post-processing) ./var/lib/systemd/catalog/database debuerreotype-0.4/scripts/debuerreotype-apt-get000077500000000000000000000007621315424740200220740ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ ' arguments' \ 'rootfs update' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] "$thisDir/debuerreotype-chroot" "$targetDir" apt-get -o Acquire::Check-Valid-Until=false "$@" debuerreotype-0.4/scripts/debuerreotype-chroot000077500000000000000000000016011315424740200220220ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ ' [args...]' \ 'rootfs apt-get update' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' cmd="${1:-}"; shift || eusage 'missing command' [ -n "$targetDir" ] epoch="$(< "$targetDir/debuerreotype-epoch")" [ -n "$epoch" ] export targetDir epoch unshare --mount bash -Eeuo pipefail -c ' [ -n "$targetDir" ] # just to be safe for dir in dev proc sys; do mount --rbind "/$dir" "$targetDir/$dir" done exec chroot "$targetDir" /usr/bin/env -i PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" TZ="$TZ" LC_ALL="$LC_ALL" SOURCE_DATE_EPOCH="$epoch" "$@" ' -- "$cmd" "$@" debuerreotype-0.4/scripts/debuerreotype-fixup000077500000000000000000000015061315424740200216630ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ 'rootfs' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] epoch="$(< "$targetDir/debuerreotype-epoch")" [ -n "$epoch" ] # https://github.com/lamby/debootstrap/commit/66b15380814aa62ca4b5807270ac57a3c8a0558d#diff-de4eef4ab836e5c6c9c1f820a2f624baR709 rm -f \ "$targetDir/var/log/dpkg.log" \ "$targetDir/var/log/bootstrap.log" \ "$targetDir/var/log/alternatives.log" \ "$targetDir/var/cache/ldconfig/aux-cache" find "$targetDir" \ -newermt "@$epoch" \ -exec touch --no-dereference --date="@$epoch" '{}' + debuerreotype-0.4/scripts/debuerreotype-gen-sources-list000077500000000000000000000022701315424740200237320ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ --flags 'deb-src' \ -- \ '[--deb-src] ' \ 'rootfs stretch http://deb.debian.org/debian http://security.debian.org' eval "$dgetopt" debSrc= while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --deb-src) debSrc=1 ;; --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' suite="${1:-}"; shift || eusage 'missing suite' mirror="${1:-}"; shift || eusage 'missing mirror' secmirror="${1:-}"; shift || eusage 'missing secmirror' [ -n "$targetDir" ] comp='main' deb() { echo "deb $*" if [ -n "$debSrc" ]; then echo "deb-src $*" fi } # https://github.com/tianon/go-aptsources/blob/e066ed9cd8cd9eef7198765bd00ec99679e6d0be/target.go#L16-L58 { case "$suite" in sid|unstable|testing) deb "$mirror" "$suite" "$comp" ;; *) deb "$mirror" "$suite" "$comp" deb "$mirror" "$suite-updates" "$comp" deb "$secmirror" "$suite/updates" "$comp" ;; esac } > "$targetDir/etc/apt/sources.list" chmod 0644 "$targetDir/etc/apt/sources.list" debuerreotype-0.4/scripts/debuerreotype-init000077500000000000000000000072151315424740200214760ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ --flags 'debian,non-debian' \ --flags 'debootstrap:' \ --flags 'debootstrap-script:' \ --flags 'keyring:,arch:' \ --flags 'merged-usr,no-merged-usr' \ -- \ ' ' \ 'rootfs stretch 2017-05-08T00:00:00Z' \ \ '--non-debian [--debootstrap-script=xyz] ' \ '--non-debian rootfs xenial http://archive.ubuntu.com/ubuntu' eval "$dgetopt" nonDebian= debootstrap= script= keyring= arch= noMergedUsr= while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --debian) nonDebian= ;; --non-debian) nonDebian=1 ;; --debootstrap) debootstrap="$1"; shift ;; --debootstrap-script) script="$1"; shift ;; --keyring) keyring="$1"; shift ;; --arch) arch="$1"; shift ;; --merged-usr) noMergedUsr= ;; --no-merged-usr) noMergedUsr=1 ;; --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] || eusage 'target-dir required' # must be non-empty if [ -e "$targetDir" ] && [ -z "$(find "$targetDir" -maxdepth 0 -empty)" ]; then echo >&2 "error: '$targetDir' already exists (and isn't empty)!" exit 1 fi suite="${1:-}"; shift || eusage 'missing suite' timestamp= mirror= secmirror= if [ -z "$nonDebian" ]; then timestamp="${1:-}"; shift || eusage 'missing timestamp' else mirror="${1:-}"; shift || eusage 'missing mirror' timestamp="$(wget -qO- "$mirror/dists/$suite/Release" | awk -F ': ' '$1 == "Date" { print $2 }')" fi epoch="$(date --date "$timestamp" '+%s')" export SOURCE_DATE_EPOCH="$epoch" if [ -z "$nonDebian" ]; then mirror="$("$thisDir/.snapshot-url.sh" "@$epoch")" secmirror="$("$thisDir/.snapshot-url.sh" "@$epoch" 'debian-security')" fi debootstrapArgs=( --force-check-gpg --variant=minbase ) [ -n "$noMergedUsr" ] || debootstrapArgs+=( --merged-usr ) [ -z "$keyring" ] || debootstrapArgs+=( --keyring="$keyring" ) [ -z "$arch" ] || debootstrapArgs+=( --arch="$arch" ) debootstrapArgs+=( "$suite" "$targetDir" "$mirror" ) [ -z "$script" ] || debootstrapArgs+=( "$script" ) "${debootstrap:-debootstrap}" "${debootstrapArgs[@]}" echo "$epoch" > "$targetDir/debuerreotype-epoch" if [ -z "$nonDebian" ]; then "$thisDir/debuerreotype-gen-sources-list" "$targetDir" "$suite" "$mirror" "$secmirror" fi # since we're minbase, we know everything included is either essential, or a dependency of essential, so let's get clean "apt-mark showmanual" output "$thisDir/debuerreotype-chroot" "$targetDir" apt-mark auto '.*' > /dev/null echo 'debuerreotype' > "$targetDir/etc/hostname" echo "$epoch" \ | md5sum \ | cut -f1 -d' ' \ > "$targetDir/etc/machine-id" # TODO should we only do this if "/etc/machine-id" already exists? { echo 'nameserver 8.8.8.8' echo 'nameserver 8.8.4.4' } > "$targetDir/etc/resolv.conf" chmod 0644 \ "$targetDir/etc/hostname" \ "$targetDir/etc/machine-id" \ "$targetDir/etc/resolv.conf" # https://bugs.debian.org/857803 # adjust field 3 in /etc/shadow and /etc/shadow- to $(( epoch / 60 / 60 / 24 )), if it's larger sp_lstchg="$(( epoch / 60 / 60 / 24 ))" for shadowFile in etc/shadow etc/shadow-; do newShadowFile="$shadowFile.debuerreotype" awk -F ':' \ -v OFS=':' \ -v sp_lstchg="$sp_lstchg" \ '{ if ($3 > sp_lstchg) { $3 = sp_lstchg } print }' "$targetDir/$shadowFile" > "$targetDir/$newShadowFile" if [ "$(< "$targetDir/$shadowFile")" != "$(< "$targetDir/$newShadowFile")" ]; then # use "cat" instead of "mv" so permissions don't change cat "$targetDir/$newShadowFile" > "$targetDir/$shadowFile" fi rm -f "$targetDir/$newShadowFile" done debuerreotype-0.4/scripts/debuerreotype-minimizing-config000077500000000000000000000121431315424740200241440ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ 'rootfs' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] # https://github.com/docker/docker/blob/d6f4fe9e38b60f63e429fff7ffced9c26cbf8236/contrib/mkimage/debootstrap#L63-L177 # prevent init scripts from running during install/update cat > "$targetDir/usr/sbin/policy-rc.d" <<-'EOF' #!/bin/sh # For most Docker users, "apt-get install" only happens during "docker build", # where starting services doesn't work and often fails in humorous ways. This # prevents those failures by stopping the services from attempting to start. exit 101 EOF chmod 0755 "$targetDir/usr/sbin/policy-rc.d" # prevent upstart scripts from running during install/update "$thisDir/debuerreotype-chroot" "$targetDir" dpkg-divert --local --rename --add /sbin/initctl > /dev/null cp -a "$targetDir/usr/sbin/policy-rc.d" "$targetDir/sbin/initctl" sed -i 's/^exit.*/exit 0/' "$targetDir/sbin/initctl" # TODO should we only do this if "/sbin/initctl" already exists? # force dpkg not to call sync() after package extraction (speeding up installs) cat > "$targetDir/etc/dpkg/dpkg.cfg.d/docker-apt-speedup" <<-'EOF' # For most Docker users, package installs happen during "docker build", which # doesn't survive power loss and gets restarted clean afterwards anyhow, so # this minor tweak gives us a nice speedup (much nicer on spinning disks, # obviously). force-unsafe-io EOF chmod 0644 "$targetDir/etc/dpkg/dpkg.cfg.d/docker-apt-speedup" # keep us lean by effectively running "apt-get clean" after every install aptGetClean='"rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true";' cat > "$targetDir/etc/apt/apt.conf.d/docker-clean" <<-EOF # Since for most Docker users, package installs happen in "docker build" steps, # they essentially become individual layers due to the way Docker handles # layering, especially using CoW filesystems. What this means for us is that # the caches that APT keeps end up just wasting space in those layers, making # our layers unnecessarily large (especially since we'll normally never use # these caches again and will instead just "docker build" again and make a brand # new image). # Ideally, these would just be invoking "apt-get clean", but in our testing, # that ended up being cyclic and we got stuck on APT's lock, so we get this fun # creation that's essentially just "apt-get clean". DPkg::Post-Invoke { $aptGetClean }; APT::Update::Post-Invoke { $aptGetClean }; Dir::Cache::pkgcache ""; Dir::Cache::srcpkgcache ""; # Note that we do realize this isn't the ideal way to do this, and are always # open to better suggestions (https://github.com/debuerreotype/debuerreotype/issues). EOF chmod 0644 "$targetDir/etc/apt/apt.conf.d/docker-clean" # remove apt-cache translations for faster "apt-get update" cat > "$targetDir/etc/apt/apt.conf.d/docker-no-languages" <<-'EOF' # In Docker, we don't often need the "Translations" files, so we're just wasting # time and space by downloading them, and this inhibits that. For users that do # need them, it's a simple matter to delete this file and "apt-get update". :) Acquire::Languages "none"; EOF chmod 0644 "$targetDir/etc/apt/apt.conf.d/docker-no-languages" cat > "$targetDir/etc/apt/apt.conf.d/docker-gzip-indexes" <<-'EOF' # Since Docker users using "RUN apt-get update && apt-get install -y ..." in # their Dockerfiles don't go delete the lists files afterwards, we want them to # be as small as possible on-disk, so we explicitly request "gz" versions and # tell Apt to keep them gzipped on-disk. # For comparison, an "apt-get update" layer without this on a pristine # "debian:wheezy" base image was "29.88 MB", where with this it was only # "8.273 MB". Acquire::GzipIndexes "true"; Acquire::CompressionTypes::Order:: "gz"; EOF chmod 0644 "$targetDir/etc/apt/apt.conf.d/docker-gzip-indexes" # update "autoremove" configuration to be aggressive about removing suggests deps that weren't manually installed cat > "$targetDir/etc/apt/apt.conf.d/docker-autoremove-suggests" <<-'EOF' # Since Docker users are looking for the smallest possible final images, the # following emerges as a very common pattern: # RUN apt-get update \ # && apt-get install -y \ # && \ # && apt-get purge -y --auto-remove # By default, APT will actually _keep_ packages installed via Recommends or # Depends if another package Suggests them, even and including if the package # that originally caused them to be installed is removed. Setting this to # "false" ensures that APT is appropriately aggressive about removing the # packages it added. # https://aptitude.alioth.debian.org/doc/en/ch02s05s05.html#configApt-AutoRemove-SuggestsImportant Apt::AutoRemove::SuggestsImportant "false"; EOF chmod 0644 "$targetDir/etc/apt/apt.conf.d/docker-autoremove-suggests" debuerreotype-0.4/scripts/debuerreotype-slimify000077500000000000000000000047531315424740200222130ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ 'rootfs' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] IFS=$'\n'; set -o noglob slimExcludes=( $(grep -vE '^#|^$' "$thisDir/.slimify-excludes" | sort -u) ) set +o noglob; unset IFS dpkgCfgFile="$targetDir/etc/dpkg/dpkg.cfg.d/docker" mkdir -p "$(dirname "$dpkgCfgFile")" { echo '# This is the "slim" variant of the Debian base image.' echo '# Many files which are normally unnecessary in containers are excluded,' echo '# and this configuration file keeps them that way.' } > "$dpkgCfgFile" neverExclude='/usr/share/doc/*/copyright' for slimExclude in "${slimExcludes[@]}"; do { echo echo "# dpkg -S '$slimExclude'" if dpkgOutput="$("$thisDir/debuerreotype-chroot" "$targetDir" dpkg -S "$slimExclude" 2>&1)"; then echo "$dpkgOutput" | sed 's/: .*//g; s/, /\n/g' | sort -u | xargs else echo "$dpkgOutput" fi | fold -w 76 -s | sed 's/^/# /' echo "path-exclude $slimExclude" } >> "$dpkgCfgFile" if [[ "$slimExclude" == *'/*' ]]; then if [ -d "$targetDir/$(dirname "$slimExclude")" ]; then # use two passes so that we don't fail trying to remove directories from $neverExclude # this is our best effort at implementing https://sources.debian.net/src/dpkg/stretch/src/filters.c/#L96-L97 in shell # step 1 -- delete everything that doesn't match "$neverExclude" and isn't a directory or a symlink "$thisDir/debuerreotype-chroot" "$targetDir" \ find "$(dirname "$slimExclude")" \ -mindepth 1 \ -not -path "$neverExclude" \ -not \( -type d -o -type l \) \ -delete # step 2 -- repeatedly delete any dangling symlinks and empty directories until there aren't any # (might have a dangling symlink in a directory which then makes it empty, or a symlink to an empty directory) while [ "$( "$thisDir/debuerreotype-chroot" "$targetDir" \ find "$(dirname "$slimExclude")" \ -mindepth 1 \( -empty -o -xtype l \) \ -delete -printf '.' \ | wc -c )" -gt 0 ]; do true; done fi else "$thisDir/debuerreotype-chroot" "$targetDir" rm -f "$slimExclude" fi done { echo echo '# always include these files, especially for license compliance' echo "path-include $neverExclude" } >> "$dpkgCfgFile" chmod 0644 "$dpkgCfgFile" debuerreotype-0.4/scripts/debuerreotype-tar000077500000000000000000000022211315424740200213110ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ --flags 'exclude:' \ --flags 'include-dev' \ -- \ '[--include-dev] ' \ 'rootfs rootfs.tar' eval "$dgetopt" excludes=() includeDev= while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --exclude) excludes+=( "$1" ); shift ;; --include-dev) includeDev=1 ;; --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] targetTar="${1:-}"; shift || eusage 'missing target-tar' [ -n "$targetTar" ] epoch="$(< "$targetDir/debuerreotype-epoch")" [ -n "$epoch" ] "$thisDir/debuerreotype-fixup" "$targetDir" tarArgs=( --create --file "$targetTar" --auto-compress --directory "$targetDir" --exclude-from "$thisDir/.tar-exclude" ) if [ -z "$includeDev" ]; then excludes+=( './dev/**' ) fi for exclude in "${excludes[@]}"; do tarArgs+=( --exclude "$exclude" ) done tarArgs+=( --numeric-owner --transform 's,^./,,' --sort name . ) tar "${tarArgs[@]}" touch --no-dereference --date="@$epoch" "$targetTar" debuerreotype-0.4/scripts/debuerreotype-version000077500000000000000000000004541315424740200222160ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ '' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done _version debuerreotype-0.4/steamos.sh000077500000000000000000000130441315424740200162510ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/scripts/.constants.sh" \ --flags 'no-build' \ -- \ '[--no-build] ' \ 'output' eval "$dgetopt" build=1 while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --no-build) build= ;; # for skipping "docker build" --) break ;; *) eusage "unknown flag '$flag'" ;; esac done outputDir="${1:-}"; shift || eusage 'missing output-dir' mkdir -p "$outputDir" outputDir="$(readlink -f "$outputDir")" securityArgs=( --cap-add SYS_ADMIN ) if docker info | grep -q apparmor; then # AppArmor blocks mount :) securityArgs+=( --security-opt apparmor=unconfined ) fi ver="$("$thisDir/scripts/debuerreotype-version")" ver="${ver%% *}" dockerImage="debuerreotype/debuerreotype:$ver" [ -z "$build" ] || docker build -t "$dockerImage" "$thisDir" steamDockerImage="$dockerImage-steamos" [ -z "$build" ] || docker build -t "$steamDockerImage" - <<-EODF FROM $dockerImage RUN wget -O valve.deb 'http://repo.steampowered.com/steamos/pool/main/v/valve-archive-keyring/valve-archive-keyring_0.5+bsos3_all.deb' \\ && apt install -y ./valve.deb \\ && rm valve.deb EODF docker run \ --rm \ "${securityArgs[@]}" \ --tmpfs /tmp:dev,exec,suid,noatime \ -w /tmp \ -e TZ='UTC' -e LC_ALL='C' \ "$steamDockerImage" \ bash -Eeuo pipefail -c ' set -x # http://repo.steampowered.com/steamos/dists/ suite="brewmaster" mirror="http://repo.steampowered.com/steamos" dpkgArch="$(dpkg --print-architecture)" exportDir="output" outputDir="$exportDir/steamos/$dpkgArch/$suite" debuerreotypeScriptsDir="$(dirname "$(readlink -f "$(which debuerreotype-init)")")" mkdir -p "$outputDir" wget -O "$outputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg" wget -O "$outputDir/Release" "$mirror/dists/$suite/Release" gpgv \ --keyring /usr/share/keyrings/valve-archive-keyring.gpg \ "$outputDir/Release.gpg" \ "$outputDir/Release" { debuerreotype-init --non-debian \ --debootstrap-script /usr/share/debootstrap/scripts/jessie \ --keyring /usr/share/keyrings/valve-archive-keyring.gpg \ --no-merged-usr \ rootfs "$suite" "$mirror" echo "deb $mirror $suite main contrib non-free" | tee rootfs/etc/apt/sources.list epoch="$(< rootfs/debuerreotype-epoch)" touch_epoch() { while [ "$#" -gt 0 ]; do local f="$1"; shift touch --no-dereference --date="@$epoch" "$f" done } debuerreotype-minimizing-config rootfs debuerreotype-apt-get rootfs update -qq debuerreotype-apt-get rootfs dist-upgrade -yqq # make a couple copies of rootfs so we can create other variants for variant in slim sbuild; do mkdir "rootfs-$variant" tar -cC rootfs . | tar -xC "rootfs-$variant" done debuerreotype-apt-get rootfs install -y --no-install-recommends iproute2 iputils-ping debuerreotype-slimify rootfs-slim # this should match the list added to the "buildd" variant in debootstrap and the list installed by sbuild # https://anonscm.debian.org/cgit/d-i/debootstrap.git/tree/scripts/sid?id=706a45681c5bba5e062a9b02e19f079cacf2a3e8#n26 # https://anonscm.debian.org/cgit/buildd-tools/sbuild.git/tree/bin/sbuild-createchroot?id=eace3d3e59e48d26eaf069d9b63a6a4c868640e6#n194 debuerreotype-apt-get rootfs-sbuild install -y --no-install-recommends build-essential fakeroot create_artifacts() { local targetBase="$1"; shift local rootfs="$1"; shift local suite="$1"; shift local variant="$1"; shift if [ "$variant" != "sbuild" ]; then debuerreotype-tar "$rootfs" "$targetBase.tar.xz" else # sbuild needs "deb-src" entries debuerreotype-chroot "$rootfs" sed -ri -e "/^deb / p; s//deb-src /" /etc/apt/sources.list # APT has odd issues with "Acquire::GzipIndexes=false" + "file://..." sources sometimes # (which are used in sbuild for "--extra-package") # Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) # ... # E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) rm -f "$rootfs/etc/apt/apt.conf.d/docker-gzip-indexes" # TODO figure out the bug and fix it in APT instead /o\ # schroot is picky about "/dev" (which is excluded by default in "debuerreotype-tar") # see https://github.com/debuerreotype/debuerreotype/pull/8#issuecomment-305855521 debuerreotype-tar --include-dev "$rootfs" "$targetBase.tar.xz" fi du -hsx "$targetBase.tar.xz" sha256sum "$targetBase.tar.xz" | cut -d" " -f1 > "$targetBase.tar.xz.sha256" touch_epoch "$targetBase.tar.xz.sha256" debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest" echo "$epoch" > "$targetBase.debuerreotype-epoch" touch_epoch "$targetBase.manifest" "$targetBase.debuerreotype-epoch" for f in debian_version os-release apt/sources.list; do targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")" cp "$rootfs/etc/$f" "$targetFile" touch_epoch "$targetFile" done } for rootfs in rootfs*/; do rootfs="${rootfs%/}" # "rootfs", "rootfs-slim", ... du -hsx "$rootfs" variant="${rootfs#rootfs}" # "", "-slim", ... variant="${variant#-}" # "", "slim", ... variantDir="$outputDir/$variant" mkdir -p "$variantDir" targetBase="$variantDir/rootfs" create_artifacts "$targetBase" "$rootfs" "$suite" "$variant" done } >&2 tar -cC "$exportDir" . ' | tar -xvC "$outputDir" debuerreotype-0.4/ubuntu.sh000077500000000000000000000136621315424740200161260ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" source "$thisDir/scripts/.constants.sh" \ --flags 'no-build' \ -- \ '[--no-build] ' \ 'output xenial' eval "$dgetopt" build=1 while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --no-build) build= ;; # for skipping "docker build" --) break ;; *) eusage "unknown flag '$flag'" ;; esac done outputDir="${1:-}"; shift || eusage 'missing output-dir' suite="${1:-}"; shift || eusage 'missing suite' mkdir -p "$outputDir" outputDir="$(readlink -f "$outputDir")" securityArgs=( --cap-add SYS_ADMIN ) if docker info | grep -q apparmor; then # AppArmor blocks mount :) securityArgs+=( --security-opt apparmor=unconfined ) fi ver="$("$thisDir/scripts/debuerreotype-version")" ver="${ver%% *}" dockerImage="debuerreotype/debuerreotype:$ver" [ -z "$build" ] || docker build -t "$dockerImage" "$thisDir" ubuntuDockerImage="$dockerImage-ubuntu" [ -z "$build" ] || docker build -t "$ubuntuDockerImage" - <<-EODF FROM $dockerImage RUN apt-get update \\ && apt-get install -y --no-install-recommends ubuntu-archive-keyring \\ && rm -rf /var/lib/apt/lists/* EODF docker run \ --rm \ "${securityArgs[@]}" \ -v /tmp \ -w /tmp \ -e suite="$suite" \ -e TZ='UTC' -e LC_ALL='C' \ "$ubuntuDockerImage" \ bash -Eeuo pipefail -c ' set -x dpkgArch="$(dpkg --print-architecture)" case "$dpkgArch" in amd64|i386) mirror="http://archive.ubuntu.com/ubuntu" secmirror="http://security.ubuntu.com/ubuntu" ;; *) mirror="http://ports.ubuntu.com/ubuntu-ports" secmirror="$mirror" # no separate security mirror for ports ;; esac exportDir="output" outputDir="$exportDir/ubuntu/$dpkgArch/$suite" debuerreotypeScriptsDir="$(dirname "$(readlink -f "$(which debuerreotype-init)")")" mkdir -p "$outputDir" wget -O "$outputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg" wget -O "$outputDir/Release" "$mirror/dists/$suite/Release" gpgv \ --keyring /usr/share/keyrings/ubuntu-archive-keyring.gpg \ "$outputDir/Release.gpg" \ "$outputDir/Release" { debuerreotype-init --non-debian \ --keyring /usr/share/keyrings/ubuntu-archive-keyring.gpg \ rootfs "$suite" "$mirror" # TODO setup proper sources.list for Ubuntu # deb http://archive.ubuntu.com/ubuntu xenial main restricted universe multiverse # deb http://archive.ubuntu.com/ubuntu xenial-updates main restricted universe multiverse # deb http://archive.ubuntu.com/ubuntu xenial-backports main restricted universe multiverse # deb http://security.ubuntu.com/ubuntu xenial-security main restricted universe multiverse epoch="$(< rootfs/debuerreotype-epoch)" touch_epoch() { while [ "$#" -gt 0 ]; do local f="$1"; shift touch --no-dereference --date="@$epoch" "$f" done } debuerreotype-minimizing-config rootfs debuerreotype-apt-get rootfs update -qq debuerreotype-apt-get rootfs dist-upgrade -yqq # make a couple copies of rootfs so we can create other variants for variant in slim sbuild; do mkdir "rootfs-$variant" tar -cC rootfs . | tar -xC "rootfs-$variant" done debuerreotype-apt-get rootfs install -y --no-install-recommends iproute2 iputils-ping debuerreotype-slimify rootfs-slim # this should match the list added to the "buildd" variant in debootstrap and the list installed by sbuild # https://anonscm.debian.org/cgit/d-i/debootstrap.git/tree/scripts/sid?id=706a45681c5bba5e062a9b02e19f079cacf2a3e8#n26 # https://anonscm.debian.org/cgit/buildd-tools/sbuild.git/tree/bin/sbuild-createchroot?id=eace3d3e59e48d26eaf069d9b63a6a4c868640e6#n194 debuerreotype-apt-get rootfs-sbuild install -y --no-install-recommends build-essential fakeroot create_artifacts() { local targetBase="$1"; shift local rootfs="$1"; shift local suite="$1"; shift local variant="$1"; shift if [ "$variant" != "sbuild" ]; then debuerreotype-tar "$rootfs" "$targetBase.tar.xz" else # sbuild needs "deb-src" entries debuerreotype-chroot "$rootfs" sed -ri -e "/^deb / p; s//deb-src /" /etc/apt/sources.list # APT has odd issues with "Acquire::GzipIndexes=false" + "file://..." sources sometimes # (which are used in sbuild for "--extra-package") # Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) # ... # E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) rm -f "$rootfs/etc/apt/apt.conf.d/docker-gzip-indexes" # TODO figure out the bug and fix it in APT instead /o\ # schroot is picky about "/dev" (which is excluded by default in "debuerreotype-tar") # see https://github.com/debuerreotype/debuerreotype/pull/8#issuecomment-305855521 debuerreotype-tar --include-dev "$rootfs" "$targetBase.tar.xz" fi du -hsx "$targetBase.tar.xz" sha256sum "$targetBase.tar.xz" | cut -d" " -f1 > "$targetBase.tar.xz.sha256" touch_epoch "$targetBase.tar.xz.sha256" debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest" echo "$epoch" > "$targetBase.debuerreotype-epoch" touch_epoch "$targetBase.manifest" "$targetBase.debuerreotype-epoch" for f in debian_version os-release apt/sources.list; do targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")" cp "$rootfs/etc/$f" "$targetFile" touch_epoch "$targetFile" done } for rootfs in rootfs*/; do rootfs="${rootfs%/}" # "rootfs", "rootfs-slim", ... du -hsx "$rootfs" variant="${rootfs#rootfs}" # "", "-slim", ... variant="${variant#-}" # "", "slim", ... variantDir="$outputDir/$variant" mkdir -p "$variantDir" targetBase="$variantDir/rootfs" create_artifacts "$targetBase" "$rootfs" "$suite" "$variant" done } >&2 tar -cC "$exportDir" . ' | tar -xvC "$outputDir"