pax_global_header00006660000000000000000000000064142576360450014526gustar00rootroot0000000000000052 comment=13a5a14f457d296f486a14713fcc3007e9e71d60 debuerreotype-0.15/000077500000000000000000000000001425763604500143315ustar00rootroot00000000000000debuerreotype-0.15/.docker-image.sh000077500000000000000000000003711425763604500172760ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(readlink -vf "$BASH_SOURCE")" thisDir="$(dirname "$thisDir")" ver="$("$thisDir/scripts/debuerreotype-version")" ver="${ver%% *}" dockerImage="debuerreotype/debuerreotype:$ver" echo "$dockerImage" debuerreotype-0.15/.dockerignore000066400000000000000000000000421425763604500170010ustar00rootroot00000000000000** !VERSION !examples/ !scripts/ debuerreotype-0.15/.github/000077500000000000000000000000001425763604500156715ustar00rootroot00000000000000debuerreotype-0.15/.github/workflows/000077500000000000000000000000001425763604500177265ustar00rootroot00000000000000debuerreotype-0.15/.github/workflows/ci.yml000066400000000000000000000115221425763604500210450ustar00rootroot00000000000000name: GitHub CI on: pull_request: push: schedule: - cron: 0 0 * * 0 workflow_dispatch: defaults: run: shell: 'bash -Eeuo pipefail -x {0}' jobs: https: name: Ensure no-TLS snapshot usage runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - name: Ensure http://snapshot.debian.org (https://github.com/debuerreotype/debuerreotype/pull/119#issuecomment-901457009) run: | rm .github/workflows/ci.yml # this file itself will always be a match, but it's the only valid one 👀 ! grep -rn 'https://snapshot.debian.org' test: strategy: matrix: include: - { SUITE: stable, CODENAME: jessie, TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: 2dfa65652d6666e1f496886a2a9b3852bd43c1df55babe7009e8dbfe4a66ba69 } - { SUITE: jessie, CODENAME: "", TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: 2dfa65652d6666e1f496886a2a9b3852bd43c1df55babe7009e8dbfe4a66ba69 } - { SUITE: testing, CODENAME: stretch, TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: f6a94df0a2eccdec7cabf576e009b639f6d011c7db62744cbc9f11b4067b5568 } - { SUITE: stretch, CODENAME: "", TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: f6a94df0a2eccdec7cabf576e009b639f6d011c7db62744cbc9f11b4067b5568 } - { SUITE: unstable, CODENAME: sid, TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: 87f46eeb98d44ff5742d87112d9cc45e51dbb1204d60cb4136b51f0edfce061f } - { SUITE: sid, CODENAME: "", TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: 87f46eeb98d44ff5742d87112d9cc45e51dbb1204d60cb4136b51f0edfce061f } - { SUITE: oldstable, CODENAME: wheezy, TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: d19b91beb77aa6d32dbf45681a8510916d70ec74d2f499ee5559fc96ada025a1 } - { SUITE: wheezy, CODENAME: "", TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: d19b91beb77aa6d32dbf45681a8510916d70ec74d2f499ee5559fc96ada025a1 } # EOL suites testing - { SUITE: eol, CODENAME: etch, TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: 893d436a060f2536f70efbdfd2e2952cf311eada558f858e6190c80b323b783e } - { SUITE: eol, CODENAME: lenny, TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: c263084bc482b1538512eb091095dd30cf55a6873b989aeee9d4e148f2f3fafa } - { SUITE: eol, CODENAME: woody, ARCH: i386, TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: f80833896e141fbfebf8c91e79da2ccca1bdeb8f8ecc4e05dd33531c32857e0f } - { SUITE: eol, CODENAME: jessie, TIMESTAMP: "2021-03-01T00:00:00Z", SHA256: 45c5553e989a8d42106029ec6f5e042bf48b29e08bf50e414f99f04c33b10fe9 } # deb822 testing - { SUITE: unstable, CODENAME: "", TIMESTAMP: "2022-06-30T00:00:00Z", SHA256: 6e781f82630cf92e68ce206a5363983be6b05fff0270de5d1839bcd5e2c415fd } - { SUITE: bookworm, CODENAME: "", TIMESTAMP: "2022-06-30T00:00:00Z", SHA256: 2c8382c35c735217e66f51fd4634bb2f4d89cd0119af2151d320ecb677cb6b20 } - { SUITE: bullseye, CODENAME: "", TIMESTAMP: "2022-06-30T00:00:00Z", SHA256: 2e65c32660eb2035874444de9e11223f34429712a7b6255f13127a3506e2bcb1 } # qemu-debootstrap testing - { ARCH: arm64, SUITE: jessie, CODENAME: "", TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: 45b3c398b472ff45399cc6cc633005f48d2359d0df8d905022d37a29434420cf } - { ARCH: sh4, SUITE: unstable, CODENAME: "", TIMESTAMP: "2022-02-01T00:00:00Z", SHA256: 8cf5ac04cd174465aff38484b0cd1b5ad83fe14585322f99a7db404b6dc02275 } - { ARCH: riscv64, SUITE: unstable, CODENAME: "", TIMESTAMP: "2022-02-01T00:00:00Z", SHA256: 3f0d1dc0cb525aa47e6d9947fa7d6e207f7fd72a1d023872a874d6202e020232 } # a few entries for "today" to try and catch issues like https://github.com/debuerreotype/debuerreotype/issues/41 sooner - { SUITE: unstable, CODENAME: "", TIMESTAMP: "today 00:00:00", SHA256: "" } - { SUITE: stable, CODENAME: "", TIMESTAMP: "today 00:00:00", SHA256: "" } - { SUITE: oldstable, CODENAME: "", TIMESTAMP: "today 00:00:00", SHA256: "" } - { DISTRO: ubuntu, SUITE: bionic } - { DISTRO: ubuntu, SUITE: focal } fail-fast: false name: Test ${{ matrix.DISTRO && format('{0} ', matrix.DISTRO) }}${{ matrix.SUITE }}${{ matrix.CODENAME && format(' ({0})', matrix.CODENAME) }}${{ matrix.ARCH && format(' [{0}]', matrix.ARCH) }}${{ matrix.TIMESTAMP && format(' at {0}', matrix.TIMESTAMP) }} runs-on: ubuntu-20.04 env: ${{ matrix }} steps: - uses: actions/checkout@v2 - name: Prepare Environment run: | sudo apt-get update -qq sudo apt-get install -yqq binfmt-support qemu-user-static docker run -d --name squignix --restart always tianon/squignix git clone --depth 1 https://github.com/tianon/pgp-happy-eyeballs.git ~/phe ~/phe/hack-my-builds.sh rm -rf ~/phe - name: Build run: | "./.validate-${DISTRO:-debian}.sh" debuerreotype-0.15/.validate-debian.sh000077500000000000000000000014361425763604500177630ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail epoch="$(TZ=UTC date --date "$TIMESTAMP" +%s)" serial="$(TZ=UTC date --date "@$epoch" +%Y%m%d)" buildArgs=() if [ "$SUITE" = 'eol' ]; then buildArgs+=( '--eol' ) SUITE="$CODENAME" elif [ -n "${CODENAME:-}" ]; then buildArgs+=( '--codename-copy' ) fi if [ -n "${ARCH:-}" ]; then buildArgs+=( "--arch=${ARCH}" ) if [ "$ARCH" != 'i386' ]; then if [ "$ARCH" != 'arm64' ]; then buildArgs+=( '--ports' ) fi fi fi buildArgs+=( validate "$SUITE" "@$epoch" ) checkFile="validate/$serial/${ARCH:-amd64}/${CODENAME:-$SUITE}/rootfs.tar.xz" mkdir -p validate set -x ./scripts/debuerreotype-version ./docker-run.sh --pull ./examples/debian.sh "${buildArgs[@]}" real="$(sha256sum "$checkFile" | cut -d' ' -f1)" [ -z "$SHA256" ] || [ "$SHA256" = "$real" ] debuerreotype-0.15/.validate-ubuntu.sh000077500000000000000000000015501425763604500200600ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail dockerImage="$(./.docker-image.sh)" dockerImage+='-ubuntu' { cat Dockerfile - <<-'EODF' RUN set -eux; \ # https://bugs.debian.org/929165 :( # http://snapshot.debian.org/package/ubuntu-keyring/ # http://snapshot.debian.org/package/ubuntu-keyring/2020.06.17.1-1/ wget -O ubuntu-keyring.deb 'http://snapshot.debian.org/archive/debian/20210307T083530Z/pool/main/u/ubuntu-keyring/ubuntu-keyring_2020.06.17.1-1_all.deb'; \ echo 'c2d8c4a9be6244bbea80c2e0e7624cbd3a2006a2 *ubuntu-keyring.deb' | sha1sum --strict --check -; \ apt-get install -y --no-install-recommends ./ubuntu-keyring.deb; \ rm ubuntu-keyring.deb EODF } | docker build --pull --tag "$dockerImage" --file - . mkdir -p validate set -x ./scripts/debuerreotype-version ./docker-run.sh --image="$dockerImage" --no-build ./examples/ubuntu.sh validate "$SUITE" debuerreotype-0.15/Dockerfile000066400000000000000000000070111425763604500163220ustar00rootroot00000000000000# docker run --cap-add SYS_ADMIN --cap-drop SETFCAP --tmpfs /tmp:dev,exec,suid,noatime ... # bootstrapping a new architecture? # ./scripts/debuerreotype-init /tmp/docker-rootfs bullseye now # ./scripts/debuerreotype-minimizing-config /tmp/docker-rootfs # ./scripts/debuerreotype-debian-sources-list /tmp/docker-rootfs bullseye # ./scripts/debuerreotype-tar /tmp/docker-rootfs - | docker import - debian:bullseye-slim # alternate: # debootstrap --variant=minbase bullseye /tmp/docker-rootfs # tar -cC /tmp/docker-rootfs . | docker import - debian:bullseye-slim # (or your own favorite set of "debootstrap" commands to create a base image for building this one FROM) FROM debian:bullseye-slim RUN set -eux; \ apt-get update; \ apt-get install -y --no-install-recommends \ debian-ports-archive-keyring \ debootstrap \ wget ca-certificates \ xz-utils \ \ gnupg dirmngr \ ; \ rm -rf /var/lib/apt/lists/* # fight the tyrrany of HSTS (which destroys our ability to transparently cache snapshot.debian.org responses) ENV WGETRC /.wgetrc RUN echo 'hsts=0' >> "$WGETRC" # https://github.com/debuerreotype/debuerreotype/issues/100 # https://tracker.debian.org/pkg/distro-info-data # http://snapshot.debian.org/package/distro-info-data/ # http://snapshot.debian.org/package/distro-info-data/0.53/ RUN set -eux; \ wget -O distro-info-data.deb 'http://snapshot.debian.org/archive/debian/20220425T210133Z/pool/main/d/distro-info-data/distro-info-data_0.53_all.deb'; \ echo '9c2044aae46ae3d54927deadaabbdfbd4c4177aa *distro-info-data.deb' | sha1sum --strict --check -; \ apt-get install -y ./distro-info-data.deb; \ rm distro-info-data.deb; \ [ -s /usr/share/distro-info/debian.csv ] # https://bugs.debian.org/973852 # https://salsa.debian.org/installer-team/debootstrap/-/merge_requests/63 # https://people.debian.org/~tianon/debootstrap-mr-63--download_main.patch RUN set -eux; \ apt-get update; \ apt-get install -y --no-install-recommends patch; \ rm -rf /var/lib/apt/lists/*; \ wget -O debootstrap-download-main.patch 'https://people.debian.org/~tianon/debootstrap-mr-63--download_main.patch'; \ echo 'ceae8f508a9b49236fa4519a44a584e6c774aa0e4446eb1551f3b69874a4cde5 *debootstrap-download-main.patch' | sha256sum --strict --check -; \ patch --input=debootstrap-download-main.patch /usr/share/debootstrap/functions; \ rm debootstrap-download-main.patch # see ".dockerignore" COPY . /opt/debuerreotype RUN set -eux; \ cd /opt/debuerreotype/scripts; \ for f in debuerreotype-*; do \ ln -svL "$PWD/$f" "/usr/local/bin/$f"; \ done; \ version="$(debuerreotype-version)"; \ [ "$version" != 'unknown' ]; \ echo "debuerreotype version $version" WORKDIR /tmp # a few example md5sum values for amd64: # debuerreotype-init --keyring /usr/share/keyrings/debian-archive-removed-keys.gpg test-stretch stretch 2017-05-08T00:00:00Z # debuerreotype-tar test-stretch test-stretch.tar # md5sum test-stretch.tar # 694f02c53651673ebe094cae3bcbb06d # ./docker-run.sh sh -euxc 'debuerreotype-init --keyring /usr/share/keyrings/debian-archive-removed-keys.gpg /tmp/rootfs stretch 2017-05-08T00:00:00Z; debuerreotype-tar /tmp/rootfs - | md5sum' # debuerreotype-init --keyring /usr/share/keyrings/debian-archive-removed-keys.gpg test-jessie jessie 2017-05-08T00:00:00Z # debuerreotype-tar test-jessie test-jessie.tar # md5sum test-jessie.tar # 354cedd99c08d213d3493a7cf0aaaad6 # ./docker-run.sh sh -euxc 'debuerreotype-init --keyring /usr/share/keyrings/debian-archive-removed-keys.gpg /tmp/rootfs jessie 2017-05-08T00:00:00Z; debuerreotype-tar /tmp/rootfs - | md5sum' debuerreotype-0.15/LICENSE000066400000000000000000000020601425763604500153340ustar00rootroot00000000000000Copyright 2017 Tianon Gravi Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. debuerreotype-0.15/README.md000066400000000000000000000166011425763604500156140ustar00rootroot00000000000000# Debuerreotype [![GitHub CI](https://github.com/debuerreotype/debuerreotype/workflows/GitHub%20CI/badge.svg?branch=master&event=push)](https://github.com/debuerreotype/debuerreotype/actions?query=workflow%3A%22GitHub+CI%22+branch%3Amaster) Reproducible, [snapshot](http://snapshot.debian.org)-based Debian rootfs builds (especially for Docker). This is based on [lamby](https://github.com/lamby)'s work for reproducible `debootstrap`: - https://github.com/lamby/debootstrap/commit/66b15380814aa62ca4b5807270ac57a3c8a0558d - https://wiki.debian.org/ReproducibleInstalls ### "Debuerreotype"? The name is an attempt at riffing off the photography basis of the word "snapshot". The [daguerreotype](https://en.wikipedia.org/wiki/Daguerreotype) process was an early method for taking photographs, and this is a method for taking "photographs" of Debian at a given point in time. ## Why? The goal is to create an auditable, reproducible process for creating rootfs tarballs (especially for use in Docker) of Debian releases, based on point-in-time snapshots from [snapshot.debian.org](http://snapshot.debian.org). However, as noted below, the only strictly Docker-specific script is `debuerreotype-minimizing-config`, which applies many configuration tweaks which are useful for Docker users and may or may not be useful outside of that context. ## Usage The usage of the scripts here center around a "rootfs" directory, which is both the working directory for building the target rootfs, and contains the `debuerreotype-epoch` file, which records our snapshot.debian.org epoch value (so we can adjust timestamps using it, as it is the basis for our reproducibility). Available scripts: | *script* | *purpose* | | --- | --- | | `debuerreotype-init` | create the initial "rootfs", given a suite and a timestamp (in some format `date(1)` can parse); `sources.list` will be pointing at snapshot.debian.org | | `debuerreotype-chroot` | run a command in the given "rootfs" (using `unshare` to mount `/dev`, `/proc`, and `/sys` from the parent environment in a simple, safe way) | | `debuerreotype-apt-get` | run `apt-get` via `debuerreotype-chroot`, including `-o Acquire::Check-Valid-Until=false` to account for older snapshots with (now) invalid `Valid-Until` values | | `debuerreotype-minimizing-config` | apply configuration tweaks to make the rootfs minimal and keep it minimal (especially targeted at Docker images, with comments explicitly describing Docker use cases) | | `debuerreotype-slimify` | remove files such as documentation to create an even smaller rootfs (used for creating `slim` variants of the Docker images, for example) | | `debuerreotype-debian-sources-list` | generate an appropriate Debian `sources.list` in the rootfs given a suite (especially for updating `sources.list` to point at deb.debian.org before generating outputs) | | `debuerreotype-recalculate-epoch` | (esp. for non-Debian) recalculate `debuerreotype-epoch` from `/var/lib/apt/lists/*_{In,}Release` files' `Date:` fields (after updating `sources.list` / `apt-get update`) | | `debuerreotype-fixup` | invoked by `debuerreotype-tar` to fixup timestamps and remove known-bad log files for determinism | | `debuerreotype-tar` | deterministically create a tar file of the rootfs | | `debuerreotype-version` | print out the version of the current `debuerreotype` installation | A simple `Dockerfile` is provided for using these scripts in a simple deterministic environment based on Docker, but given a recent enough version of `debootstrap`, they should run fine outside Docker as well (and their deterministic properties have been verified on at least a Gentoo host in addition to the provided Debian-based Docker environment). The provided `Dockerfile` also includes comments with hints for bootstrapping the environment on a new architecture (which then presumably doesn't have a `debian` Docker base image yet). Full example: (see [`examples/debian.sh`](examples/debian.sh) for this in practice) ```console $ debuerreotype-init --keyring /usr/share/keyrings/debian-archive-removed-keys.gpg rootfs stretch 2017-01-01T00:00:00Z I: Retrieving InRelease I: Checking Release signature I: Valid Release signature (key id 126C0D24BD8A2942CC7DF8AC7638D0442B90D010) ... I: Checking component main on http://snapshot.debian.org/archive/debian/20170101T000000Z... ... I: Base system installed successfully. $ cat rootfs/debuerreotype-epoch 1483228800 $ debuerreotype-minimizing-config rootfs $ debuerreotype-apt-get rootfs update -qq $ debuerreotype-apt-get rootfs dist-upgrade -yqq $ debuerreotype-apt-get rootfs install -yqq --no-install-recommends inetutils-ping iproute2 debconf: delaying package configuration, since apt-utils is not installed Selecting previously unselected package libelf1:amd64. (Reading database ... 6299 files and directories currently installed.) Preparing to unpack .../0-libelf1_0.166-2.2_amd64.deb ... Unpacking libelf1:amd64 (0.166-2.2) ... Selecting previously unselected package libmnl0:amd64. Preparing to unpack .../1-libmnl0_1.0.4-2_amd64.deb ... Unpacking libmnl0:amd64 (1.0.4-2) ... Selecting previously unselected package iproute2. Preparing to unpack .../2-iproute2_4.9.0-1_amd64.deb ... Unpacking iproute2 (4.9.0-1) ... Selecting previously unselected package netbase. Preparing to unpack .../3-netbase_5.3_all.deb ... Unpacking netbase (5.3) ... Selecting previously unselected package inetutils-ping. Preparing to unpack .../4-inetutils-ping_2%3a1.9.4-2+b1_amd64.deb ... Unpacking inetutils-ping (2:1.9.4-2+b1) ... Setting up libelf1:amd64 (0.166-2.2) ... Processing triggers for libc-bin (2.24-8) ... Setting up libmnl0:amd64 (1.0.4-2) ... Setting up netbase (5.3) ... Setting up inetutils-ping (2:1.9.4-2+b1) ... Setting up iproute2 (4.9.0-1) ... Processing triggers for libc-bin (2.24-8) ... $ debuerreotype-debian-sources-list rootfs stretch $ debuerreotype-tar rootfs - | sha256sum e6f10da22f7ab5996f855c85ad5ae38cd786029c57893436c3bb2320f30bc188 - $ # try it! you should get that same sha256sum value! ``` (As a one-liner via [`docker-run.sh`](docker-run.sh): `./docker-run.sh sh -euxc 'debuerreotype-init --keyring /usr/share/keyrings/debian-archive-removed-keys.gpg /tmp/rootfs stretch 2017-01-01T00:00:00Z; debuerreotype-minimizing-config /tmp/rootfs; debuerreotype-apt-get /tmp/rootfs update -qq; debuerreotype-apt-get /tmp/rootfs dist-upgrade -yqq; debuerreotype-apt-get /tmp/rootfs install -yqq --no-install-recommends inetutils-ping iproute2; debuerreotype-debian-sources-list /tmp/rootfs stretch; debuerreotype-tar /tmp/rootfs - | sha256sum'`) ## How much have you verified this? Well, I ran the scripts across seven explicit architectures (`amd64`, `arm64`, `armel`, `armhf`, `i386`, `ppc64el`, `s390x`) and eight explicit suites (`oldstable`, `stable`, `testing`, `unstable`, `wheezy`, `jessie`, `stretch`, `sid`) for a timestamp of `2017-05-16T00:00:00Z` (where supported, since `wheezy`/`oldstable` didn't or no longer currently supports some of those architectures), and there were no modifications to any of the tarballs after several runs across several days. Additionally, GitHub Actions runs with a fixed timestamp value across several suites to verify that their checksums are reproducible, as expected. From time to time, comments in the files generated by `debuerreotype-minimizing-config` might change (for example), which would obviously result in a different checksum, but a simple [`diffoscope`](https://diffoscope.org/) should be sufficient to verify that the change is benign. debuerreotype-0.15/VERSION000066400000000000000000000000051425763604500153740ustar00rootroot000000000000000.15 debuerreotype-0.15/docker-run.sh000077500000000000000000000036051425763604500167450ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail # usage: mkdir -p output && ./run-script.sh ./examples/debian.sh output ... thisDir="$(readlink -vf "$BASH_SOURCE")" thisDir="$(dirname "$thisDir")" source "$thisDir/scripts/.constants.sh" \ --flags 'image:' \ --flags 'no-bind' \ --flags 'no-build' \ --flags 'pull' \ -- \ '[--image=foo/bar:baz] [--no-build] [--no-bind] [--pull] [script/command]' \ './examples/debian.sh output stretch 2017-05-08T00:00:00Z --no-build --image=debuerreotype:ubuntu ./examples/ubuntu.sh output xenial' eval "$dgetopt" image= build=1 bindMount=1 pull= while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --image) image="$1"; shift ;; --no-bind) bindMount= ;; --no-build) build= ;; --pull) pull=1 ;; --) break ;; *) eusage "unknown flag '$flag'" ;; esac done if [ -z "$image" ]; then image="$("$thisDir/.docker-image.sh")" fi if [ -n "$build" ]; then docker build ${pull:+--pull} --tag "$image" "$thisDir" elif [ -n "$pull" ]; then docker pull "$image" else # make sure "docker run" doesn't pull (we have `--no-build` and no explicit `--pull`) docker image inspect "$image" > /dev/null fi args=( --hostname debuerreotype --init --interactive --rm # we ought to be able to mount/unshare --cap-add SYS_ADMIN # make sure we don't get extended attributes --cap-drop SETFCAP # AppArmor also blocks mount/unshare :) --security-opt apparmor=unconfined # --debian-eol potato wants to run "chroot ... mount ... /proc" which gets blocked (i386, ancient binaries, blah blah blah) --security-opt seccomp=unconfined # (other arches see this occasionally too) --tmpfs /tmp:dev,exec,suid,noatime --env TMPDIR=/tmp --workdir /workdir ) if [ -n "$bindMount" ]; then args+=( --mount "type=bind,src=$PWD,dst=/workdir" ) else args+=( --volume /workdir ) fi if [ -t 0 ] && [ -t 1 ]; then args+=( --tty ) fi exec docker run "${args[@]}" "$image" "$@" debuerreotype-0.15/examples/000077500000000000000000000000001425763604500161475ustar00rootroot00000000000000debuerreotype-0.15/examples/debian-all.sh000077500000000000000000000073771425763604500205140ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail suites=( unstable testing stable oldstable oldoldstable # just in case (will no-op with "not supported on 'arch'" unless it exists) oldoldoldstable ) debuerreotypeScriptsDir="$(which debuerreotype-init)" debuerreotypeScriptsDir="$(readlink -vf "$debuerreotypeScriptsDir")" debuerreotypeScriptsDir="$(dirname "$debuerreotypeScriptsDir")" source "$debuerreotypeScriptsDir/.constants.sh" \ --flags 'arch:' \ -- \ '[--arch=] ' \ 'output 2017-05-08T00:00:00Z' eval "$dgetopt" arch= while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --arch) arch="$1"; shift ;; --) break ;; *) eusage "unknown flag '$flag'" ;; esac done outputDir="${1:-}"; shift || eusage 'missing output-dir' timestamp="${1:-}"; shift || eusage 'missing timestamp' debianArgs=( --codename-copy ) mirror="$("$debuerreotypeScriptsDir/.snapshot-url.sh" "$timestamp")" secmirror="$("$debuerreotypeScriptsDir/.snapshot-url.sh" "$timestamp" 'debian-security')" dpkgArch="${arch:-$(dpkg --print-architecture | awk -F- '{ print $NF }')}" echo echo "-- BUILDING TARBALLS FOR '$dpkgArch' FROM '$mirror/' --" echo debianArgs+=( --arch="$dpkgArch" ) thisDir="$(readlink -vf "$BASH_SOURCE")" thisDir="$(dirname "$thisDir")" _eol-date() { local codename="$1"; shift # "bullseye", "buster", etc. if [ ! -s /usr/share/distro-info/debian.csv ]; then echo >&2 "warning: looks like we are missing 'distro-info-data' (/usr/share/distro-info/debian.csv); cannot calculate EOL dates accurately!" exit 1 fi awk -F, -v codename="$codename" ' NR == 1 { headers = NF for (i = 1; i <= headers; i++) { header[i] = $i } next } { delete row for (i = 1; i <= NF && i <= headers; i++) { row[header[i]] = $i } } row["series"] == codename { if (row["eol-lts"] != "") { eol = row["eol-lts"] exit 0 } if (row["eol"] != "") { eol = row["eol"] exit 0 } exit 1 } END { if (eol != "") { print eol exit 0 } exit 1 } ' /usr/share/distro-info/debian.csv } _codename() { local dist="$1"; shift local release if release="$(wget --quiet --output-document=- "$mirror/dists/$dist/InRelease")"; then : elif release="$(wget --quiet --output-document=- "$mirror/dists/$dist/Release")"; then : else return 1 fi local codename codename="$(awk '$1 == "Codename:" { print $2 }' <<<"$release")" [ -n "$codename" ] || return 1 echo "$codename" } _check() { local host="$1"; shift # "$mirror", "$secmirror" local dist="$1"; shift # "$suite-security", "$suite/updates", "$suite" local comp="${1:-main}" if wget --quiet --spider "$host/dists/$dist/$comp/binary-$dpkgArch/Packages.xz"; then return 0 fi if wget --quiet --spider "$host/dists/$dist/$comp/binary-$dpkgArch/Packages.gz"; then return 0 fi return 1 } for suite in "${suites[@]}"; do doSkip= case "$suite" in testing | unstable) ;; *) # https://lists.debian.org/debian-devel-announce/2019/07/msg00004.html if \ ! _check "$secmirror" "$suite-security" \ && ! _check "$secmirror" "$suite/updates" \ ; then doSkip=1 fi if [ -z "$doSkip" ] && codename="$(_codename "$suite")" && eol="$(_eol-date "$codename")"; then epoch="$(date --date "$timestamp" '+%s')" eolEpoch="$(date --date "$eol" '+%s')" if [ "$epoch" -ge "$eolEpoch" ]; then echo >&2 echo >&2 "warning: '$suite' ('$codename') is EOL at '$timestamp' ('$eol'); skipping" echo >&2 continue fi fi ;; esac if ! _check "$mirror" "$suite"; then doSkip=1 fi if [ -n "$doSkip" ]; then echo >&2 echo >&2 "warning: '$suite' not supported on '$dpkgArch' (at '$timestamp'); skipping" echo >&2 continue fi "$thisDir/debian.sh" "${debianArgs[@]}" "$outputDir" "$suite" "$timestamp" done debuerreotype-0.15/examples/debian.sh000077500000000000000000000330001425763604500177240ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail debuerreotypeScriptsDir="$(which debuerreotype-init)" debuerreotypeScriptsDir="$(readlink -vf "$debuerreotypeScriptsDir")" debuerreotypeScriptsDir="$(dirname "$debuerreotypeScriptsDir")" source "$debuerreotypeScriptsDir/.constants.sh" \ --flags 'codename-copy' \ --flags 'eol,ports' \ --flags 'arch:' \ --flags 'include:,exclude:' \ -- \ '[--codename-copy] [--eol] [--ports] [--arch=] ' \ 'output stretch 2017-05-08T00:00:00Z --codename-copy output stable 2017-05-08T00:00:00Z --eol output squeeze 2016-03-14T00:00:00Z --eol --arch i386 output sarge 2016-03-14T00:00:00Z' eval "$dgetopt" codenameCopy= eol= ports= include= exclude= arch= while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --codename-copy) codenameCopy=1 ;; # for copying a "stable.tar.xz" to "stretch.tar.xz" with updated sources.list (saves a lot of extra building work) --eol) eol=1 ;; # for using "archive.debian.org" --ports) ports=1 ;; # for using "debian-ports" --arch) arch="$1"; shift ;; # for adding "--arch" to debuerreotype-init --include) include="${include:+$include,}$1"; shift ;; --exclude) exclude="${exclude:+$exclude,}$1"; shift ;; --) break ;; *) eusage "unknown flag '$flag'" ;; esac done outputDir="${1:-}"; shift || eusage 'missing output-dir' suite="${1:-}"; shift || eusage 'missing suite' timestamp="${1:-}"; shift || eusage 'missing timestamp' set -x outputDir="$(readlink -ve "$outputDir")" tmpDir="$(mktemp --directory --tmpdir "debuerreotype.$suite.XXXXXXXXXX")" trap "$(printf 'rm -rf %q' "$tmpDir")" EXIT export TZ='UTC' LC_ALL='C' epoch="$(date --date "$timestamp" +%s)" serial="$(date --date "@$epoch" +%Y%m%d)" dpkgArch="${arch:-$(dpkg --print-architecture | awk -F- '{ print $NF }')}" exportDir="$tmpDir/output" archDir="$exportDir/$serial/$dpkgArch" tmpOutputDir="$archDir/$suite" touch_epoch() { while [ "$#" -gt 0 ]; do local f="$1"; shift touch --no-dereference --date="@$epoch" "$f" done } for archive in '' security; do snapshotUrlFile="$archDir/snapshot-url${archive:+-${archive}}" mirrorArgs=() if [ -n "$ports" ]; then mirrorArgs+=( --ports ) fi if [ -n "$eol" ]; then mirrorArgs+=( --eol ) fi mirrorArgs+=( "@$epoch" "$suite${archive:+-$archive}" "$dpkgArch" main ) if ! mirrors="$("$debuerreotypeScriptsDir/.debian-mirror.sh" "${mirrorArgs[@]}")"; then if [ "$archive" = 'security' ]; then # if we fail to find the security mirror, we're probably not security supported (which is ~fine) continue else exit 1 fi fi eval "$mirrors" [ -n "$snapshotMirror" ] snapshotUrlDir="$(dirname "$snapshotUrlFile")" mkdir -p "$snapshotUrlDir" echo "$snapshotMirror" > "$snapshotUrlFile" touch_epoch "$snapshotUrlFile" done initArgs=( --arch "$dpkgArch" ) if [ -z "$eol" ]; then initArgs+=( --debian ) else initArgs+=( --debian-eol ) fi if [ -n "$ports" ]; then initArgs+=( --debian-ports --include=debian-ports-archive-keyring ) fi export GNUPGHOME="$tmpDir/gnupg" mkdir -p "$GNUPGHOME" keyring="$tmpDir/debian-archive-$suite-keyring.gpg" if [ "$suite" = 'slink' ]; then # slink (2.1) introduced apt, but without PGP 😅 initArgs+=( --no-check-gpg ) elif [ "$suite" = 'potato' ]; then # src:debian-archive-keyring was created in 2006, thus does not include a key for potato (2.2; EOL in 2003) gpg --batch --no-default-keyring --keyring "$keyring" \ --keyserver keyserver.ubuntu.com \ --recv-keys 8FD47FF1AA9372C37043DC28AA7DEB7B722F1AED initArgs+=( --keyring "$keyring" ) else # check against all releases (ie, combine both "debian-archive-keyring.gpg" and "debian-archive-removed-keys.gpg"), since we cannot really know whether the target release became EOL later than the snapshot date we are targeting gpg --batch --no-default-keyring --keyring "$keyring" --import \ /usr/share/keyrings/debian-archive-keyring.gpg \ /usr/share/keyrings/debian-archive-removed-keys.gpg if [ -n "$ports" ]; then gpg --batch --no-default-keyring --keyring "$keyring" --import \ /usr/share/keyrings/debian-ports-archive-keyring.gpg \ /usr/share/keyrings/debian-ports-archive-keyring-removed.gpg fi initArgs+=( --keyring "$keyring" ) fi mkdir -p "$tmpOutputDir" mirror="$(< "$archDir/snapshot-url")" if [ -f "$keyring" ] && wget -O "$tmpOutputDir/InRelease" "$mirror/dists/$suite/InRelease"; then gpgv \ --keyring "$keyring" \ --output "$tmpOutputDir/Release" \ "$tmpOutputDir/InRelease" [ -s "$tmpOutputDir/Release" ] elif [ -f "$keyring" ] && wget -O "$tmpOutputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg" && wget -O "$tmpOutputDir/Release" "$mirror/dists/$suite/Release"; then rm -f "$tmpOutputDir/InRelease" # remove wget leftovers gpgv \ --keyring "$keyring" \ "$tmpOutputDir/Release.gpg" \ "$tmpOutputDir/Release" [ -s "$tmpOutputDir/Release" ] elif [ "$suite" = 'slink' ]; then # "Release" files were introduced in potato (2.2+) rm -f "$tmpOutputDir/InRelease" "$tmpOutputDir/Release.gpg" "$tmpOutputDir/Release" # remove wget leftovers else echo >&2 "error: failed to fetch either InRelease or Release.gpg+Release for '$suite' (from '$mirror')" exit 1 fi codename= if [ -f "$tmpOutputDir/Release" ]; then codename="$(awk -F ': ' '$1 == "Codename" { print $2; exit }' "$tmpOutputDir/Release")" fi if [ -n "$codenameCopy" ] && [ "$codename" = "$suite" ]; then # if codename already is the same as suite, then making a copy does not make any sense codenameCopy= fi if [ -n "$codenameCopy" ] && [ -z "$codename" ]; then echo >&2 "error: --codename-copy specified but we failed to get a Codename for $suite" exit 1 fi initArgs+=( # disable merged-usr (for now?) due to the following compelling arguments: # - https://bugs.debian.org/src:usrmerge ("dpkg-query" breaks, etc) # - https://bugs.debian.org/914208 ("buildd" variant disables merged-usr still) # - https://github.com/debuerreotype/docker-debian-artifacts/issues/60#issuecomment-461426406 --no-merged-usr ) if [ -n "$include" ]; then initArgs+=( --include="$include" ) fi if [ -n "$exclude" ]; then initArgs+=( --exclude="$exclude" ) fi rootfsDir="$tmpDir/rootfs" debuerreotype-init "${initArgs[@]}" "$rootfsDir" "$suite" "@$epoch" aptVersion="$("$debuerreotypeScriptsDir/.apt-version.sh" "$rootfsDir")" # regenerate sources.list to make the deb822/line-based opinion explicit # https://lists.debian.org/debian-devel/2021/11/msg00026.html sourcesListArgs=() [ -z "$eol" ] || sourcesListArgs+=( --eol ) [ -z "$ports" ] || sourcesListArgs+=( --ports ) if dpkg --compare-versions "$aptVersion" '>=' '2.3~' && { [ "$suite" = 'unstable' ] || [ "$suite" = 'sid' ]; }; then # just unstable for now (TODO after some time testing this, we should update this to bookworm+ which is aptVersion 2.3+) sourcesListArgs+=( --deb822 ) sourcesListFile='/etc/apt/sources.list.d/debian.sources' else sourcesListArgs+=( --no-deb822 ) sourcesListFile='/etc/apt/sources.list' fi debuerreotype-debian-sources-list "${sourcesListArgs[@]}" --snapshot "$rootfsDir" "$suite" [ -s "$rootfsDir$sourcesListFile" ] # trust, but verify if [ -n "$eol" ]; then debuerreotype-gpgv-ignore-expiration-config "$rootfsDir" fi debuerreotype-minimizing-config "$rootfsDir" debuerreotype-apt-get "$rootfsDir" update -qq if dpkg --compare-versions "$aptVersion" '>=' '1.1~'; then debuerreotype-apt-get "$rootfsDir" full-upgrade -yqq else debuerreotype-apt-get "$rootfsDir" dist-upgrade -yqq fi if dpkg --compare-versions "$aptVersion" '>=' '0.7.14~'; then # https://salsa.debian.org/apt-team/apt/commit/06d79436542ccf3e9664306da05ba4c34fba4882 noInstallRecommends='--no-install-recommends' else # etch (4.0) and lower do not support --no-install-recommends noInstallRecommends='-o APT::Install-Recommends=0' fi if [ -n "$eol" ] && dpkg --compare-versions "$aptVersion" '>=' '0.7.26~'; then # https://salsa.debian.org/apt-team/apt/commit/1ddb859611d2e0f3d9ea12085001810f689e8c99 echo 'Acquire::Check-Valid-Until "false";' > "$rootfsDir"/etc/apt/apt.conf.d/check-valid-until.conf # TODO make this a real script so it can have a nice comment explaining why we do it for EOL releases? fi # copy the rootfs to create other variants mkdir "$rootfsDir"-slim tar -cC "$rootfsDir" . | tar -xC "$rootfsDir"-slim # for historical reasons (related to their usefulness in debugging non-working container networking in container early days before "--network container:xxx"), Debian 10 and older non-slim images included both "ping" and "ip" above "minbase", but in 11+ (Bullseye), that will no longer be the case and we will instead be a faithful minbase again :D epoch2021="$(date --date '2021-01-01 00:00:00' +%s)" if [ "$epoch" -lt "$epoch2021" ] || { isDebianBusterOrOlder="$([ -f "$rootfsDir/etc/os-release" ] && source "$rootfsDir/etc/os-release" && [ -n "${VERSION_ID:-}" ] && [ "${VERSION_ID%%.*}" -le 10 ] && echo 1)" && [ -n "$isDebianBusterOrOlder" ]; }; then # prefer iproute2 if it exists iproute=iproute2 if ! debuerreotype-apt-get "$rootfsDir" install -qq -s iproute2 &> /dev/null; then # poor wheezy iproute=iproute fi ping=iputils-ping if debuerreotype-chroot "$rootfsDir" bash -c 'command -v ping > /dev/null'; then # if we already have "ping" (as in potato, 2.2), skip installing any extra ping package ping= fi debuerreotype-apt-get "$rootfsDir" install -y $noInstallRecommends $ping $iproute fi debuerreotype-slimify "$rootfsDir"-slim _sanitize_basename() { local f="$1"; shift f="$(basename "$f")" f="$(sed -r -e 's/[^a-zA-Z0-9_-]+/-/g' <<<"$f")" echo "$f" } sourcesListBase="$(_sanitize_basename "$sourcesListFile")" create_artifacts() { local targetBase="$1"; shift local rootfs="$1"; shift local suite="$1"; shift local variant="$1"; shift # make a copy of the snapshot-facing sources.list file before we overwrite it cp "$rootfs$sourcesListFile" "$targetBase.$sourcesListBase-snapshot" touch_epoch "$targetBase.$sourcesListBase-snapshot" debuerreotype-debian-sources-list "${sourcesListArgs[@]}" "$rootfs" "$suite" local tarArgs=( # https://www.freedesktop.org/software/systemd/man/machine-id.html --exclude ./etc/machine-id # "debuerreotype-fixup" will make this an empty file for reproducibility, but for our Docker images it seems more appropriate for it to not exist (since they've never actually been "booted" so having the "first boot" logic trigger if someone were to run systemd in them conceptually makes sense) ) case "$suite" in sarge) # 3.1 # for some reason, sarge creates "/var/cache/man/index.db" with some obvious embedded unix timestamps (but if we exclude it, "man" still works properly, so *shrug*) tarArgs+=( --exclude ./var/cache/man/index.db ) ;; woody) # 3.0 # woody not only contains "exim", but launches it during our build process and tries to email "root@debuerreotype" (which fails and creates non-reproducibility) tarArgs+=( --exclude ./var/spool/exim --exclude ./var/log/exim ) ;; potato) # 2.2 tarArgs+=( # for some reason, pototo leaves a core dump (TODO figure out why??) --exclude './core' # also, it leaves some junk in /tmp (/tmp/fdmount.conf.tmp.XXX) --exclude './tmp/fdmount.conf.tmp.*' ) ;; slink) # 2.1 tarArgs+=( # same as potato :( --exclude './tmp/fdmount.conf.tmp.*' ) ;; esac debuerreotype-tar "${tarArgs[@]}" "$rootfs" "$targetBase.tar.xz" du -hsx "$targetBase.tar.xz" sha256sum "$targetBase.tar.xz" | cut -d' ' -f1 > "$targetBase.tar.xz.sha256" touch_epoch "$targetBase.tar.xz.sha256" debuerreotype-chroot "$rootfs" bash -c ' if ! dpkg-query -W 2> /dev/null; then # --debian-eol woody has no dpkg-query dpkg -l fi ' > "$targetBase.manifest" echo "$suite" > "$targetBase.apt-dist" echo "$dpkgArch" > "$targetBase.dpkg-arch" echo "$epoch" > "$targetBase.debuerreotype-epoch" echo "$variant" > "$targetBase.debuerreotype-variant" debuerreotype-version > "$targetBase.debuerreotype-version" touch_epoch "$targetBase".{manifest,apt-dist,dpkg-arch,debuerreotype-*} for f in /etc/debian_version /etc/os-release "$sourcesListFile"; do targetFile="$(_sanitize_basename "$f")" targetFile="$targetBase.$targetFile" if [ -e "$rootfs$f" ]; then # /etc/os-release does not exist in --debian-eol squeeze, for example (hence the existence check) cp "$rootfs$f" "$targetFile" touch_epoch "$targetFile" fi done } for rootfs in "$rootfsDir"*/; do rootfs="${rootfs%/}" # "../rootfs", "../rootfs-slim", ... du -hsx "$rootfs" variant="$(basename "$rootfs")" # "rootfs", "rootfs-slim", ... variant="${variant#rootfs}" # "", "-slim", ... variant="${variant#-}" # "", "slim", ... variantDir="$tmpOutputDir/$variant" mkdir -p "$variantDir" targetBase="$variantDir/rootfs" create_artifacts "$targetBase" "$rootfs" "$suite" "$variant" done if [ -n "$codenameCopy" ]; then codenameDir="$archDir/$codename" mkdir -p "$codenameDir" tar -cC "$tmpOutputDir" --exclude='**/rootfs.*' . | tar -xC "$codenameDir" for rootfs in "$rootfsDir"*/; do rootfs="${rootfs%/}" # "../rootfs", "../rootfs-slim", ... variant="$(basename "$rootfs")" # "rootfs", "rootfs-slim", ... variant="${variant#rootfs}" # "", "-slim", ... variant="${variant#-}" # "", "slim", ... variantDir="$codenameDir/$variant" targetBase="$variantDir/rootfs" # point sources.list back at snapshot.debian.org temporarily (but this time pointing at $codename instead of $suite) debuerreotype-debian-sources-list --snapshot "${sourcesListArgs[@]}" "$rootfs" "$codename" create_artifacts "$targetBase" "$rootfs" "$codename" "$variant" done fi user="$(stat --format '%u' "$outputDir")" group="$(stat --format '%g' "$outputDir")" tar --create --directory="$exportDir" --owner="$user" --group="$group" . | tar --extract --verbose --directory="$outputDir" debuerreotype-0.15/examples/oci-image.sh000077500000000000000000000205101425763604500203360ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail # create https://github.com/opencontainers/image-spec/blob/v1.0.1/image-layout.md (+ Docker's "manifest.json") from the output of "debian.sh" # the resulting file is suitable for "ctr image import" or "docker load" # (this can *technically* run via "docker-run.sh", but IMO it's much easier to run unprivileged on the host) # RUN apt-get update \ # && apt-get install -y jq pigz \ # && rm -rf /var/lib/apt/lists/* thisDir="$(readlink -vf "$BASH_SOURCE")" thisDir="$(dirname "$thisDir")" if [ -x "$thisDir/../scripts/debuerreotype-init" ]; then debuerreotypeScriptsDir="$(dirname "$thisDir")/scripts" else debuerreotypeScriptsDir="$(which debuerreotype-init)" debuerreotypeScriptsDir="$(readlink -vf "$debuerreotypeScriptsDir")" debuerreotypeScriptsDir="$(dirname "$debuerreotypeScriptsDir")" fi source "$debuerreotypeScriptsDir/.constants.sh" \ --flags 'meta:' \ -- \ ' ' \ 'out/oci-unstable-slim.tar out/20210511/amd64/unstable/slim' eval "$dgetopt" meta= while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --meta) meta="$1"; shift ;; --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetFile="${1:-}"; shift || eusage 'missing target-file' # "something.tar" sourceDir="${1:-}"; shift || eusage 'missing source-directory' # "out/YYYYMMDD/ARCH/SUITE{,/slim}" targetFile="$(readlink -vf "$targetFile")" if [ -n "$meta" ]; then meta="$(readlink -vf "$meta")" fi sourceDir="$(readlink -ve "$sourceDir")" tempDir="$(mktemp -d)" trap "$(printf 'rm -rf %q' "$tempDir")" EXIT mkdir -p "$tempDir/oci/blobs/sha256" jq -ncS '{ imageLayoutVersion: "1.0.0" }' > "$tempDir/oci/oci-layout" version="$(< "$sourceDir/rootfs.debuerreotype-version")" epoch="$(< "$sourceDir/rootfs.debuerreotype-epoch")" iso8601="$(date --date="@$epoch" '+%Y-%m-%dT%H:%M:%SZ')" export version epoch iso8601 if [ -s "$sourceDir/rootfs.apt-dist" ]; then suite="$(< "$sourceDir/rootfs.apt-dist")" # TODO remove this fallback once debuerreotype 0.13 is released and we can safely assume "rootfs.apt-dist" exists else suite="$(awk '$1 == "deb" { print $3; exit }' "$sourceDir/rootfs.sources-list")" fi export suite if [ -s "$sourceDir/rootfs.debuerreotype-variant" ]; then variant="$(< "$sourceDir/rootfs.debuerreotype-variant")" # TODO remove this fallback once debuerreotype 0.13 is released and we can safely assume "rootfs.debuerreotype-variant" exists else dirBase="$(basename "$sourceDir")" case "$dirBase" in slim) variant="$dirBase" ;; "$suite") variant='' ;; *) echo >&2 "error: unknown variant: '$variant'"; exit 1 ;; esac fi if [ -s "$sourceDir/rootfs.dpkg-arch" ]; then dpkgArch="$(< "$sourceDir/rootfs.dpkg-arch")" # TODO remove these fallbacks once debuerreotype 0.13 is released and we can safely assume "rootfs.dpkg-arch" exists elif [ -n "$variant" ]; then # xxx/YYYYMMDD/ARCH/SUITE/slim dpkgArch="$(cd "$sourceDir/../.." && basename "$PWD")" else # xxx/YYYYMMDD/ARCH/SUITE dpkgArch="$(cd "$sourceDir/.." && basename "$PWD")" fi unset goArch goArm= case "$dpkgArch" in amd64 | arm64 | s390x | riscv64) goArch="$dpkgArch" ;; armel | arm) goArch='arm'; goArm='5' ;; armhf) goArch='arm'; if grep -qi raspbian "$sourceDir/rootfs.os-release"; then goArm='6'; else goArm='7'; fi ;; i386) goArch='386' ;; mips64el | ppc64el) goArch="${dpkgArch%el}le" ;; *) echo >&2 "error: unknown dpkg architecture: '$dpkgArch'"; exit 1 ;; esac unset bashbrewArch case "$goArch" in 386) bashbrewArch='i386' ;; amd64 | mips64le | ppc64le | riscv64 | s390x) bashbrewArch="$goArch" ;; arm) bashbrewArch="${goArch}32v${goArm}" ;; arm64) bashbrewArch="${goArch}v8" ;; *) echo >&2 "error: unknown Go architecture: '$goArch'"; exit 1 ;; esac export dpkgArch goArch goArm bashbrewArch osID="$(id="$(grep -E '^ID=' "$sourceDir/rootfs.os-release")" && eval "$id" && echo "${ID:-}")" || : # "debian", "raspbian", "ubuntu", etc : "${osID:=debian}" # if for some reason the above fails, fall back to "debian" echo >&2 "processing $osID '$suite'${variant:+", variant '$variant'"}, architecture '$dpkgArch' ('$bashbrewArch')" _sha256() { sha256sum "$@" | cut -d' ' -f1 } echo >&2 "decompressing rootfs (xz) ..." xz --decompress --threads=0 --stdout "$sourceDir/rootfs.tar.xz" > "$tempDir/rootfs.tar" diffId="$(_sha256 "$tempDir/rootfs.tar")" export diffId="sha256:$diffId" echo >&2 "recompressing rootfs (gzip) ..." pigz --best --no-time "$tempDir/rootfs.tar" rootfsSize="$(stat --format='%s' "$tempDir/rootfs.tar.gz")" rootfsSha256="$(_sha256 "$tempDir/rootfs.tar.gz")" export rootfsSize rootfsSha256 mv "$tempDir/rootfs.tar.gz" "$tempDir/oci/blobs/sha256/$rootfsSha256" script='debian.sh' if [ -x "$thisDir/$osID.sh" ]; then script="$osID.sh" fi export script echo >&2 "generating config ..." # https://github.com/opencontainers/image-spec/blob/v1.0.1/config.md jq -ncS ' { config: { Env: [ "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ], Entrypoint: [], Cmd: [ "bash" ], }, created: env.iso8601, history: [ { created: env.iso8601, created_by: ( "# " + env.script + if env.script != "raspbian.sh" then " --arch " + (env.dpkgArch | @sh) else "" end + " out/ " + (env.suite | @sh) + if env.script == "debian.sh" then " " else " # " end + ("@" + env.epoch | @sh) ), comment: ( "debuerreotype " + env.version ), } ], rootfs: { type: "layers", diff_ids: [ env.diffId ], }, os: "linux", architecture: env.goArch, } + if env.goArch == "arm64" then { variant: "v8" } elif env.goArch == "arm" then { variant: ( "v" + env.goArm ) } else {} end ' > "$tempDir/config.json" configSize="$(stat --format='%s' "$tempDir/config.json")" configSha256="$(_sha256 "$tempDir/config.json")" export configSize configSha256 mv "$tempDir/config.json" "$tempDir/oci/blobs/sha256/$configSha256" # https://github.com/opencontainers/image-spec/blob/v1.0.1/manifest.md jq -ncS ' { schemaVersion: 2, mediaType: "application/vnd.docker.distribution.manifest.v2+json", config: { mediaType: "application/vnd.docker.container.image.v1+json", size: (env.configSize | tonumber), digest: ( "sha256:" + env.configSha256 ), }, layers: [ { mediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip", size: (env.rootfsSize | tonumber), digest: ( "sha256:" + env.rootfsSha256 ), } ], } ' > "$tempDir/manifest.json" manifestSize="$(stat --format='%s' "$tempDir/manifest.json")" manifestSha256="$(_sha256 "$tempDir/manifest.json")" export manifestSize manifestSha256 mv "$tempDir/manifest.json" "$tempDir/oci/blobs/sha256/$manifestSha256" export repo="$bashbrewArch/$osID" # "amd64/debian", "arm32v6/raspbian", etc. export tag="$suite${variant:+-$variant}" # "buster", "buster-slim", etc. export image="$repo:$tag" # https://github.com/opencontainers/image-spec/blob/v1.0.1/image-index.md jq -ncS ' { schemaVersion: 2, manifests: [ { mediaType: "application/vnd.docker.distribution.manifest.v2+json", size: (env.manifestSize | tonumber), digest: ( "sha256:" + env.manifestSha256 ), annotations: { "io.containerd.image.name": env.image, "org.opencontainers.image.ref.name": env.tag, }, } ], } ' > "$tempDir/oci/index.json" # Docker's "manifest.json" so that we can "docker load" the result of this script too jq -ncS ' [ { Config: ( "blobs/sha256/" + env.configSha256 ), Layers: [ "blobs/sha256/" + env.rootfsSha256 ], RepoTags: [ env.image ], } ] ' > "$tempDir/oci/manifest.json" echo >&2 "fixing timestamps ..." find "$tempDir/oci" \ -newermt "@$epoch" \ -exec touch --no-dereference --date="@$epoch" '{}' + echo >&2 "generating tarball ($targetFile) ..." tar --create \ --auto-compress \ --directory "$tempDir/oci" \ --file "$targetFile" \ --numeric-owner --owner 1000:1000 \ --sort name \ . touch --no-dereference --date="@$epoch" "$targetFile" platform="$(jq -c 'with_entries(select(.key == ([ "os", "architecture", "variant" ][])))' "$tempDir/oci/blobs/sha256/$configSha256")" jq -n --argjson platform "$platform" ' { image: env.image, repo: env.repo, tag: env.tag, id: ( "sha256:" + env.configSha256 ), digest: ( "sha256:" + env.manifestSha256 ), platform: $platform, } ' > "$tempDir/meta.json" touch --no-dereference --date="@$epoch" "$tempDir/meta.json" if [ -n "$meta" ]; then cp -a "$tempDir/meta.json" "$meta" fi jq >&2 . "$tempDir/meta.json" debuerreotype-0.15/examples/raspbian.sh000077500000000000000000000150761425763604500203160ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail # # https://archive.raspbian.org/raspbian/pool/main/r/raspbian-archive-keyring/ # RUN wget -O raspbian.deb 'https://archive.raspbian.org/raspbian/pool/main/r/raspbian-archive-keyring/raspbian-archive-keyring-udeb_20120528.2_all.udeb' \ # && apt-get install -y ./raspbian.deb \ # && rm raspbian.deb debuerreotypeScriptsDir="$(which debuerreotype-init)" debuerreotypeScriptsDir="$(readlink -vf "$debuerreotypeScriptsDir")" debuerreotypeScriptsDir="$(dirname "$debuerreotypeScriptsDir")" source "$debuerreotypeScriptsDir/.constants.sh" \ -- \ ' ' \ 'output stretch' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done outputDir="${1:-}"; shift || eusage 'missing output-dir' suite="${1:-}"; shift || eusage 'missing suite' set -x outputDir="$(readlink -ve "$outputDir")" tmpDir="$(mktemp --directory --tmpdir "debuerreotype.$suite.XXXXXXXXXX")" trap "$(printf 'rm -rf %q' "$tmpDir")" EXIT export TZ='UTC' LC_ALL='C' dpkgArch='armhf' exportDir="$tmpDir/output" archDir="$exportDir/raspbian/$dpkgArch" tmpOutputDir="$archDir/$suite" #mirror='http://archive.raspbian.org/raspbian' mirror='http://mirrordirector.raspbian.org/raspbian' # (https://www.raspbian.org/RaspbianMirrors#The_mirror_redirection_system) initArgs=( --arch "$dpkgArch" --non-debian ) export GNUPGHOME="$tmpDir/gnupg" mkdir -p "$GNUPGHOME" keyring='/usr/share/keyrings/raspbian-archive-keyring.gpg' if [ ! -s "$keyring" ]; then # since we're using mirrors, we ought to be more explicit about download verification keyUrl='https://archive.raspbian.org/raspbian.public.key' ( set +x echo >&2 echo >&2 "WARNING: missing '$keyring' (from 'raspbian-archive-keyring' package)" echo >&2 " downloading '$keyUrl' (without verification beyond TLS)!" echo >&2 ) sleep 5 keyring="$tmpDir/raspbian-archive-keyring.gpg" wget -O "$keyring.asc" "$keyUrl" gpg --batch --no-default-keyring --keyring "$keyring" --import "$keyring.asc" rm -f "$keyring.asc" fi initArgs+=( --keyring "$keyring" ) mkdir -p "$tmpOutputDir" if [ -f "$keyring" ] && wget -O "$tmpOutputDir/InRelease" "$mirror/dists/$suite/InRelease"; then gpgv \ --keyring "$keyring" \ --output "$tmpOutputDir/Release" \ "$tmpOutputDir/InRelease" [ -s "$tmpOutputDir/Release" ] elif [ -f "$keyring" ] && wget -O "$tmpOutputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg" && wget -O "$tmpOutputDir/Release" "$mirror/dists/$suite/Release"; then rm -f "$tmpOutputDir/InRelease" # remove wget leftovers gpgv \ --keyring "$keyring" \ "$tmpOutputDir/Release.gpg" \ "$tmpOutputDir/Release" [ -s "$tmpOutputDir/Release" ] else echo >&2 "error: failed to fetch either InRelease or Release.gpg+Release for '$suite' (from '$mirror')" exit 1 fi initArgs+=( # disable merged-usr (for now?) due to the following compelling arguments: # - https://bugs.debian.org/src:usrmerge ("dpkg-query" breaks, etc) # - https://bugs.debian.org/914208 ("buildd" variant disables merged-usr still) # - https://github.com/debuerreotype/docker-debian-artifacts/issues/60#issuecomment-461426406 --no-merged-usr ) rootfsDir="$tmpDir/rootfs" debuerreotype-init "${initArgs[@]}" "$rootfsDir" "$suite" "$mirror" debuerreotype-minimizing-config "$rootfsDir" # TODO do we need to update sources.list here? (security?) debuerreotype-apt-get "$rootfsDir" update -qq debuerreotype-recalculate-epoch "$rootfsDir" epoch="$(< "$rootfsDir/debuerreotype-epoch")" touch_epoch() { while [ "$#" -gt 0 ]; do local f="$1"; shift touch --no-dereference --date="@$epoch" "$f" done } aptVersion="$("$debuerreotypeScriptsDir/.apt-version.sh" "$rootfsDir")" if dpkg --compare-versions "$aptVersion" '>=' '1.1~'; then debuerreotype-apt-get "$rootfsDir" full-upgrade -yqq else debuerreotype-apt-get "$rootfsDir" dist-upgrade -yqq fi # copy the rootfs to create other variants mkdir "$rootfsDir"-slim tar -cC "$rootfsDir" . | tar -xC "$rootfsDir"-slim # for historical reasons (related to their usefulness in debugging non-working container networking in container early days before "--network container:xxx"), Debian 10 and older non-slim images included both "ping" and "ip" above "minbase", but in 11+ (Bullseye), that will no longer be the case and we will instead be a faithful minbase again :D epoch2021="$(date --date '2021-01-01 00:00:00' +%s)" if [ "$epoch" -lt "$epoch2021" ] || { isDebianBusterOrOlder="$([ -f "$rootfsDir/etc/os-release" ] && source "$rootfsDir/etc/os-release" && [ -n "${VERSION_ID:-}" ] && [ "${VERSION_ID%%.*}" -le 10 ] && echo 1)" && [ -n "$isDebianBusterOrOlder" ]; }; then # prefer iproute2 if it exists iproute=iproute2 if ! debuerreotype-apt-get "$rootfsDir" install -qq -s iproute2 &> /dev/null; then # poor wheezy iproute=iproute fi ping=iputils-ping noInstallRecommends='--no-install-recommends' debuerreotype-apt-get "$rootfsDir" install -y $noInstallRecommends $ping $iproute fi debuerreotype-slimify "$rootfsDir"-slim create_artifacts() { local targetBase="$1"; shift local rootfs="$1"; shift local suite="$1"; shift local variant="$1"; shift local tarArgs=() debuerreotype-tar "${tarArgs[@]}" "$rootfs" "$targetBase.tar.xz" du -hsx "$targetBase.tar.xz" sha256sum "$targetBase.tar.xz" | cut -d' ' -f1 > "$targetBase.tar.xz.sha256" touch_epoch "$targetBase.tar.xz.sha256" debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest" echo "$suite" > "$targetBase.apt-dist" echo "$dpkgArch" > "$targetBase.dpkg-arch" echo "$epoch" > "$targetBase.debuerreotype-epoch" echo "$variant" > "$targetBase.debuerreotype-variant" debuerreotype-version > "$targetBase.debuerreotype-version" touch_epoch "$targetBase".{manifest,apt-dist,dpkg-arch,debuerreotype-*} for f in debian_version os-release apt/sources.list; do targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")" if [ -e "$rootfs/etc/$f" ]; then cp "$rootfs/etc/$f" "$targetFile" touch_epoch "$targetFile" fi done } for rootfs in "$rootfsDir"*/; do rootfs="${rootfs%/}" # "../rootfs", "../rootfs-slim", ... du -hsx "$rootfs" variant="$(basename "$rootfs")" # "rootfs", "rootfs-slim", ... variant="${variant#rootfs}" # "", "-slim", ... variant="${variant#-}" # "", "slim", ... variantDir="$tmpOutputDir/$variant" mkdir -p "$variantDir" targetBase="$variantDir/rootfs" create_artifacts "$targetBase" "$rootfs" "$suite" "$variant" done user="$(stat --format '%u' "$outputDir")" group="$(stat --format '%g' "$outputDir")" tar --create --directory="$exportDir" --owner="$user" --group="$group" . | tar --extract --verbose --directory="$outputDir" debuerreotype-0.15/examples/steamos.sh000077500000000000000000000132001425763604500201550ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail # # https://repo.steampowered.com/steamos/pool/main/v/valve-archive-keyring/?C=M&O=D # RUN wget -O valve.deb 'https://repo.steampowered.com/steamos/pool/main/v/valve-archive-keyring/valve-archive-keyring_0.6+bsosc2_all.deb' \ # && apt-get install -y ./valve.deb \ # && rm valve.deb debuerreotypeScriptsDir="$(which debuerreotype-init)" debuerreotypeScriptsDir="$(readlink -vf "$debuerreotypeScriptsDir")" debuerreotypeScriptsDir="$(dirname "$debuerreotypeScriptsDir")" source "$debuerreotypeScriptsDir/.constants.sh" \ --flags 'arch:' \ -- \ '[--arch=] ' \ 'output' eval "$dgetopt" arch= while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --arch) arch="$1"; shift ;; # for adding "--arch" to debuerreotype-init --) break ;; *) eusage "unknown flag '$flag'" ;; esac done outputDir="${1:-}"; shift || eusage 'missing output-dir' suite="${1:-brewmaster}" # http://repo.steampowered.com/steamos/dists/ set -x outputDir="$(readlink -ve "$outputDir")" tmpDir="$(mktemp --directory --tmpdir "debuerreotype.$suite.XXXXXXXXXX")" trap "$(printf 'rm -rf %q' "$tmpDir")" EXIT export TZ='UTC' LC_ALL='C' dpkgArch="${arch:-$(dpkg --print-architecture | awk -F- '{ print $NF }')}" exportDir="$tmpDir/output" archDir="$exportDir/steamos/$dpkgArch" tmpOutputDir="$archDir/$suite" mirror='http://repo.steampowered.com/steamos' initArgs=( --arch "$dpkgArch" --non-debian --debootstrap-script jessie --include valve-archive-keyring --exclude debian-archive-keyring ) keyring='/usr/share/keyrings/valve-archive-keyring.gpg' if [ -f "$keyring" ]; then initArgs+=( --keyring "$keyring" ) else initArgs+=( --no-check-gpg ) fi mkdir -p "$tmpOutputDir" if [ -f "$keyring" ] && wget -O "$tmpOutputDir/InRelease" "$mirror/dists/$suite/InRelease"; then gpgv \ --keyring "$keyring" \ --output "$tmpOutputDir/Release" \ "$tmpOutputDir/InRelease" [ -s "$tmpOutputDir/Release" ] elif [ -f "$keyring" ] && wget -O "$tmpOutputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg" && wget -O "$tmpOutputDir/Release" "$mirror/dists/$suite/Release"; then rm -f "$tmpOutputDir/InRelease" # remove wget leftovers gpgv \ --keyring "$keyring" \ "$tmpOutputDir/Release.gpg" \ "$tmpOutputDir/Release" [ -s "$tmpOutputDir/Release" ] else rm -f "$tmpOutputDir/InRelease" "$tmpOutputDir/Release.gpg" "$tmpOutputDir/Release" # remove wget leftovers echo >&2 "warning: failed to fetch either InRelease or Release.gpg+Release for '$suite' (from '$mirror')" fi initArgs+=( # disable merged-usr (for now?) due to the following compelling arguments: # - https://bugs.debian.org/src:usrmerge ("dpkg-query" breaks, etc) # - https://bugs.debian.org/914208 ("buildd" variant disables merged-usr still) # - https://github.com/debuerreotype/docker-debian-artifacts/issues/60#issuecomment-461426406 --no-merged-usr ) rootfsDir="$tmpDir/rootfs" debuerreotype-init "${initArgs[@]}" "$rootfsDir" "$suite" "$mirror" debuerreotype-minimizing-config "$rootfsDir" echo "deb $mirror $suite main contrib non-free" | tee "$rootfsDir/etc/apt/sources.list" debuerreotype-apt-get "$rootfsDir" update -qq debuerreotype-recalculate-epoch "$rootfsDir" epoch="$(< "$rootfsDir/debuerreotype-epoch")" touch_epoch() { while [ "$#" -gt 0 ]; do local f="$1"; shift touch --no-dereference --date="@$epoch" "$f" done } touch_epoch "$rootfsDir/etc/apt/sources.list" aptVersion="$("$debuerreotypeScriptsDir/.apt-version.sh" "$rootfsDir")" if dpkg --compare-versions "$aptVersion" '>=' '1.1~'; then debuerreotype-apt-get "$rootfsDir" full-upgrade -yqq else debuerreotype-apt-get "$rootfsDir" dist-upgrade -yqq fi # copy the rootfs to create other variants mkdir "$rootfsDir"-slim tar -cC "$rootfsDir" . | tar -xC "$rootfsDir"-slim # prefer iproute2 if it exists iproute=iproute2 if ! debuerreotype-apt-get "$rootfsDir" install -qq -s iproute2 &> /dev/null; then # poor wheezy iproute=iproute fi debuerreotype-apt-get "$rootfsDir" install -y --no-install-recommends iputils-ping $iproute debuerreotype-slimify "$rootfsDir"-slim create_artifacts() { local targetBase="$1"; shift local rootfs="$1"; shift local suite="$1"; shift local variant="$1"; shift local tarArgs=() debuerreotype-tar "${tarArgs[@]}" "$rootfs" "$targetBase.tar.xz" du -hsx "$targetBase.tar.xz" sha256sum "$targetBase.tar.xz" | cut -d' ' -f1 > "$targetBase.tar.xz.sha256" touch_epoch "$targetBase.tar.xz.sha256" debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest" echo "$suite" > "$targetBase.apt-dist" echo "$dpkgArch" > "$targetBase.dpkg-arch" echo "$epoch" > "$targetBase.debuerreotype-epoch" echo "$variant" > "$targetBase.debuerreotype-variant" debuerreotype-version > "$targetBase.debuerreotype-version" touch_epoch "$targetBase".{manifest,apt-dist,dpkg-arch,debuerreotype-*} for f in debian_version os-release apt/sources.list; do targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")" if [ -e "$rootfs/etc/$f" ]; then cp "$rootfs/etc/$f" "$targetFile" touch_epoch "$targetFile" fi done } for rootfs in "$rootfsDir"*/; do rootfs="${rootfs%/}" # "../rootfs", "../rootfs-slim", ... du -hsx "$rootfs" variant="$(basename "$rootfs")" # "rootfs", "rootfs-slim", ... variant="${variant#rootfs}" # "", "-slim", ... variant="${variant#-}" # "", "slim", ... variantDir="$tmpOutputDir/$variant" mkdir -p "$variantDir" targetBase="$variantDir/rootfs" create_artifacts "$targetBase" "$rootfs" "$suite" "$variant" done user="$(stat --format '%u' "$outputDir")" group="$(stat --format '%g' "$outputDir")" tar --create --directory="$exportDir" --owner="$user" --group="$group" . | tar --extract --verbose --directory="$outputDir" debuerreotype-0.15/examples/ubuntu.sh000077500000000000000000000134641425763604500200400ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail # RUN apt-get update \ # && apt-get install -y ubuntu-keyring \ # && rm -rf /var/lib/apt/lists/* debuerreotypeScriptsDir="$(which debuerreotype-init)" debuerreotypeScriptsDir="$(readlink -vf "$debuerreotypeScriptsDir")" debuerreotypeScriptsDir="$(dirname "$debuerreotypeScriptsDir")" source "$debuerreotypeScriptsDir/.constants.sh" \ --flags 'arch:' \ -- \ '[--arch=] ' \ 'output xenial --arch arm64 output bionic' eval "$dgetopt" arch= while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --arch) arch="$1"; shift ;; # for adding "--arch" to debuerreotype-init --) break ;; *) eusage "unknown flag '$flag'" ;; esac done outputDir="${1:-}"; shift || eusage 'missing output-dir' suite="${1:-}"; shift || eusage 'missing suite' set -x outputDir="$(readlink -ve "$outputDir")" tmpDir="$(mktemp --directory --tmpdir "debuerreotype.$suite.XXXXXXXXXX")" trap "$(printf 'rm -rf %q' "$tmpDir")" EXIT export TZ='UTC' LC_ALL='C' dpkgArch="${arch:-$(dpkg --print-architecture | awk -F- '{ print $NF }')}" exportDir="$tmpDir/output" archDir="$exportDir/ubuntu/$dpkgArch" tmpOutputDir="$archDir/$suite" case "$dpkgArch" in amd64 | i386) mirror='http://archive.ubuntu.com/ubuntu' secmirror='http://security.ubuntu.com/ubuntu' ;; *) mirror='http://ports.ubuntu.com/ubuntu-ports' secmirror="$mirror" # no separate security mirror for ports ;; esac initArgs=( --arch "$dpkgArch" --non-debian ) keyring='/usr/share/keyrings/ubuntu-archive-keyring.gpg' initArgs+=( --keyring "$keyring" ) mkdir -p "$tmpOutputDir" if [ -f "$keyring" ] && wget -O "$tmpOutputDir/InRelease" "$mirror/dists/$suite/InRelease"; then gpgv \ --keyring "$keyring" \ --output "$tmpOutputDir/Release" \ "$tmpOutputDir/InRelease" [ -s "$tmpOutputDir/Release" ] elif [ -f "$keyring" ] && wget -O "$tmpOutputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg" && wget -O "$tmpOutputDir/Release" "$mirror/dists/$suite/Release"; then rm -f "$tmpOutputDir/InRelease" # remove wget leftovers gpgv \ --keyring "$keyring" \ "$tmpOutputDir/Release.gpg" \ "$tmpOutputDir/Release" [ -s "$tmpOutputDir/Release" ] else rm -f "$tmpOutputDir/InRelease" "$tmpOutputDir/Release.gpg" "$tmpOutputDir/Release" # remove wget leftovers echo >&2 "error: failed to fetch either InRelease or Release.gpg+Release for '$suite' (from '$mirror')" exit 1 fi initArgs+=( # disable merged-usr (for now?) due to the following compelling arguments: # - https://bugs.debian.org/src:usrmerge ("dpkg-query" breaks, etc) # - https://bugs.debian.org/914208 ("buildd" variant disables merged-usr still) # - https://github.com/debuerreotype/docker-debian-artifacts/issues/60#issuecomment-461426406 --no-merged-usr ) rootfsDir="$tmpDir/rootfs" debuerreotype-init "${initArgs[@]}" "$rootfsDir" "$suite" "$mirror" debuerreotype-minimizing-config "$rootfsDir" # setup "proper" sources.list tee "$rootfsDir/etc/apt/sources.list" <<-EOS deb $mirror $suite main restricted universe multiverse deb $mirror $suite-updates main restricted universe multiverse deb $mirror $suite-backports main restricted universe multiverse deb $secmirror $suite-security main restricted universe multiverse EOS # TODO make components list a script flag? backports? debuerreotype-apt-get "$rootfsDir" update -qq debuerreotype-recalculate-epoch "$rootfsDir" epoch="$(< "$rootfsDir/debuerreotype-epoch")" touch_epoch() { while [ "$#" -gt 0 ]; do local f="$1"; shift touch --no-dereference --date="@$epoch" "$f" done } touch_epoch "$rootfsDir/etc/apt/sources.list" aptVersion="$("$debuerreotypeScriptsDir/.apt-version.sh" "$rootfsDir")" if dpkg --compare-versions "$aptVersion" '>=' '1.1~'; then debuerreotype-apt-get "$rootfsDir" full-upgrade -yqq else debuerreotype-apt-get "$rootfsDir" dist-upgrade -yqq fi # copy the rootfs to create other variants mkdir "$rootfsDir"-slim tar -cC "$rootfsDir" . | tar -xC "$rootfsDir"-slim # prefer iproute2 if it exists iproute=iproute2 if ! debuerreotype-apt-get "$rootfsDir" install -qq -s iproute2 &> /dev/null; then # poor wheezy iproute=iproute fi debuerreotype-apt-get "$rootfsDir" install -y --no-install-recommends iputils-ping $iproute debuerreotype-slimify "$rootfsDir"-slim create_artifacts() { local targetBase="$1"; shift local rootfs="$1"; shift local suite="$1"; shift local variant="$1"; shift local tarArgs=() debuerreotype-tar "${tarArgs[@]}" "$rootfs" "$targetBase.tar.xz" du -hsx "$targetBase.tar.xz" sha256sum "$targetBase.tar.xz" | cut -d' ' -f1 > "$targetBase.tar.xz.sha256" touch_epoch "$targetBase.tar.xz.sha256" debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest" echo "$suite" > "$targetBase.apt-dist" echo "$dpkgArch" > "$targetBase.dpkg-arch" echo "$epoch" > "$targetBase.debuerreotype-epoch" echo "$variant" > "$targetBase.debuerreotype-variant" debuerreotype-version > "$targetBase.debuerreotype-version" touch_epoch "$targetBase".{manifest,apt-dist,dpkg-arch,debuerreotype-*} for f in debian_version os-release apt/sources.list; do targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")" if [ -e "$rootfs/etc/$f" ]; then cp "$rootfs/etc/$f" "$targetFile" touch_epoch "$targetFile" fi done } for rootfs in "$rootfsDir"*/; do rootfs="${rootfs%/}" # "../rootfs", "../rootfs-slim", ... du -hsx "$rootfs" variant="$(basename "$rootfs")" # "rootfs", "rootfs-slim", ... variant="${variant#rootfs}" # "", "-slim", ... variant="${variant#-}" # "", "slim", ... variantDir="$tmpOutputDir/$variant" mkdir -p "$variantDir" targetBase="$variantDir/rootfs" create_artifacts "$targetBase" "$rootfs" "$suite" "$variant" done user="$(stat --format '%u' "$outputDir")" group="$(stat --format '%g' "$outputDir")" tar --create --directory="$exportDir" --owner="$user" --group="$group" . | tar --extract --verbose --directory="$outputDir" debuerreotype-0.15/scripts/000077500000000000000000000000001425763604500160205ustar00rootroot00000000000000debuerreotype-0.15/scripts/.apt-version.sh000077500000000000000000000024451425763604500207110ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ 'rootfs' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] package="${1:-apt}" # if dpkg-query does not exist, we must be on woody or older, so just assume something ancient (suggested version is the one in woody, since it should be old enough for any "fancy" features we're using this to exclude) fallback= case "$package" in apt) fallback='0.5.4' ;; # woody dpkg) fallback='1.9.21' ;; # woody esac # scrape package versions so we can do some basic feature detection (especially to remove unsupported settings on --debian-eol) "$thisDir/debuerreotype-chroot" "$targetDir" bash -c ' package="$1"; shift fallback="$1"; shift if command -v dpkg-query &> /dev/null; then dpkg-query --show --showformat "\${Version}\n" "$package" elif [ -n "$fallback" ]; then # if dpkg-query does not exist, we must be on woody or older echo "$fallback" else echo >&2 "error: missing dpkg-query and no fallback defined in debuerreotype for $package" exit 1 fi ' -- "$package" "$fallback" debuerreotype-0.15/scripts/.constants.sh000066400000000000000000000036201425763604500204470ustar00rootroot00000000000000#!/usr/bin/env bash # constants of the universe export TZ='UTC' LC_ALL='C' umask 0002 scriptsDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" self="$(basename "$0")" options="$(getopt -n "$BASH_SOURCE" -o '+' --long 'flags:,flags-short:' -- "$@")" dFlags='help,version' dFlagsShort='h?' usageStr= __cgetopt() { eval "set -- $options" # in a function since otherwise "set" will overwrite the parent script's positional args too unset options while true; do local flag="$1"; shift case "$flag" in --flags) dFlags="${dFlags:+$dFlags,}$1"; shift ;; --flags-short) dFlagsShort="${dFlagsShort}$1"; shift ;; --) break ;; *) echo >&2 "error: unexpected $BASH_SOURCE flag '$flag'"; exit 1 ;; esac done while [ "$#" -gt 0 ]; do local IFS=$'\n' local usagePrefix='usage:' usageLine= for usageLine in $1; do usageStr+="$usagePrefix $self${usageLine:+ $usageLine}"$'\n' usagePrefix=' ' done usagePrefix=' ie:' for usageLine in $2; do usageStr+="$usagePrefix $self${usageLine:+ $usageLine}"$'\n' usagePrefix=' ' done usageStr+=$'\n' shift 2 done } __cgetopt _version() { local v if [ -r "$scriptsDir/../VERSION" ]; then v="$(< "$scriptsDir/../VERSION")" else v='unknown' fi if [ -d "$scriptsDir/../.git" ] && command -v git > /dev/null; then local commit="$(git -C "$scriptsDir" rev-parse --short 'HEAD^{commit}')" v="$v commit $commit" fi echo "$v" } usage() { echo -n "$usageStr" local v="$(_version)" echo "debuerreotype version $v" } eusage() { if [ "$#" -gt 0 ]; then echo >&2 "error: $*"$'\n' fi usage >&2 exit 1 } _dgetopt() { getopt -n "$self" \ -o "+$dFlagsShort" \ --long "$dFlags" \ -- "$@" \ || eusage 'getopt failed' } dgetopt='options="$(_dgetopt "$@")"; eval "set -- $options"; unset options' dgetopt-case() { local flag="$1"; shift case "$flag" in -h|'-?'|--help) usage; exit 0 ;; --version) _version; exit 0 ;; esac } debuerreotype-0.15/scripts/.debian-mirror.sh000077500000000000000000000062751425763604500212010ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ --flags 'eol,ports' \ -- \ '[--eol] [--ports] ' \ '--eol 2021-03-01T00:00:00Z jessie amd64 main 2021-03-01T00:00:00Z buster-security arm64 main --ports 2021-03-01T00:00:00Z sid riscv64 main' eval "$dgetopt" eol= ports= while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --eol) eol=1 ;; --ports) ports=1 ;; --) break ;; *) eusage "unknown flag '$flag'" ;; esac done timestamp="${1:-}"; shift || eusage 'missing timestamp' suite="${1:-}"; shift || eusage 'missing suite' arch="${1:-}"; shift || eusage 'missing arch' component="${1:-}"; shift || eusage 'missing component' if [[ "$suite" == *-security ]]; then target='security' else target='standard' fi epoch="$(date --date "$timestamp" '+%s')" if [ -z "$ports" ]; then standardMirrors=( 'http://deb.debian.org/debian' ) snapshotStandardMirrors=( "$("$thisDir/.snapshot-url.sh" "@$epoch")" ) else standardMirrors=( 'http://deb.debian.org/debian-ports' ) snapshotStandardMirrors=( "$("$thisDir/.snapshot-url.sh" "@$epoch" 'debian-ports')" ) fi securityMirrors=( 'http://deb.debian.org/debian-security' ) snapshotSecurityMirrors=( "$("$thisDir/.snapshot-url.sh" "@$epoch" 'debian-security')" ) if [ -n "$eol" ]; then archiveSnapshotMirror="$("$thisDir/.snapshot-url.sh" "@$epoch" 'debian-archive')" standardMirrors=( 'http://archive.debian.org/debian' "${standardMirrors[@]}" ) snapshotStandardMirrors=( "$archiveSnapshotMirror/debian" "${snapshotStandardMirrors[@]}" ) securityMirrors=( 'http://archive.debian.org/debian-security' "${securityMirrors[@]}" ) snapshotSecurityMirrors=( "$archiveSnapshotMirror/debian-security" "${snapshotSecurityMirrors[@]}" ) fi case "$target" in standard) nonSnapshotMirrors=( "${standardMirrors[@]}" ) snapshotMirrors=( "${snapshotStandardMirrors[@]}" ) ;; security) nonSnapshotMirrors=( "${securityMirrors[@]}" ) snapshotMirrors=( "${snapshotSecurityMirrors[@]}" ) ;; *) echo >&2 "error: unknown target: '$target'"; exit 1 ;; esac _find() { local findSuite="${1:-$suite}" local i for i in "${!snapshotMirrors[@]}"; do local mirror="${nonSnapshotMirrors[$i]}" snapshotMirror="${snapshotMirrors[$i]}" # http://snapshot.debian.org/archive/debian-archive/20160314T000000Z/debian/dists/squeeze-updates/main/binary-amd64/Packages.gz if \ wget --quiet --spider -O /dev/null -o /dev/null "$snapshotMirror/dists/$findSuite/$component/binary-$arch/Packages.xz" \ || wget --quiet --spider -O /dev/null -o /dev/null "$snapshotMirror/dists/$findSuite/$component/binary-$arch/Packages.gz" \ ; then declare -g mirror="$mirror" snapshotMirror="$snapshotMirror" suite="$findSuite" return 0 fi done if [ "$target" = 'security' ] && [[ "$findSuite" == *-security ]]; then if _find "${suite%-security}/updates"; then return 0 fi fi return 1 } if ! _find; then echo >&2 "warning: no apparent '$suite/$component' for '$arch' on any of the following" for mirror in "${snapshotMirrors[@]}"; do echo >&2 " - $mirror"; done exit 1 fi printf 'mirror=%q\nsnapshotMirror=%q\nfoundSuite=%q\n' "$mirror" "$snapshotMirror" "$suite" debuerreotype-0.15/scripts/.debootstrap-scripts/000077500000000000000000000000001425763604500221115ustar00rootroot00000000000000debuerreotype-0.15/scripts/.debootstrap-scripts/potato000066400000000000000000000103071425763604500233430ustar00rootroot00000000000000# # This script was taken from debootstrap version 1.0.125 and then hacked to avoid invoking "init u" in postinst scripts (which tries to send SIGHUP to PID 1). # mirror_style release download_style apt var-state default_mirror http://archive.debian.org/debian force_md5 LIBC=libc6 if [ "$ARCH" = alpha ]; then LIBC="libc6.1" fi work_out_debs () { required="base-files base-passwd bash bsdutils debconf-tiny debianutils diff dpkg e2fsprogs fileutils findutils grep gzip hostname ldso libc6 libdb2 libgdbmg1 libncurses5 libnewt0 libpam-modules libpam-runtime libpam0g libpopt0 libreadline4 libstdc++2.10 login makedev mawk modutils mount ncurses-base ncurses-bin passwd perl-5.005-base perl-base procps sed shellutils slang1 sysklogd sysvinit tar textutils update util-linux whiptail" base="adduser ae apt base-config elvis-tiny fbset fdutils gettext-base console-data console-tools console-tools-libs libdb2 libwrap0 locales modconf netbase ftp ppp pppconfig pump tasksel tcpd textutils telnet xviddetect" without_package () { echo "$2" | tr ' ' '\n' | grep -v "^$1$" | tr '\n' ' ' } case $ARCH in "alpha") required="$(without_package "libc6" "$required") libc6.1" ;; "i386") base="$base fdflush isapnptools lilo mbr pciutils pcmcia-cs psmisc setserial syslinux" ;; *) # other arches may have special needs not yet represented here # oh well, Potato is old esac } first_stage_install () { extract $required :> "$TARGET/var/lib/dpkg/status" echo > "$TARGET/var/lib/dpkg/available" setup_etc echo '# UNCONFIGURED FSTAB FOR BASE SYSTEM' > "$TARGET/etc/fstab" chown 0:0 "$TARGET/etc/fstab"; chmod 644 "$TARGET/etc/fstab" x_feign_install () { local pkg=$1 local deb="$(debfor $pkg)" local ver="$(extract_deb_field "$TARGET/$deb" Version)" mkdir -p "$TARGET/var/lib/dpkg/info" echo \ "Package: $pkg Version: $ver Status: install ok installed" >> "$TARGET/var/lib/dpkg/status" touch "$TARGET/var/lib/dpkg/info/${pkg}.list" } setup_devices x_feign_install dpkg if [ -e "$TARGET/usr/bin/perl-5.005.dist" ]; then mv "$TARGET/usr/bin/perl-5.005.dist" "$TARGET/usr/bin/perl-5.005" fi if [ ! -e "$TARGET/usr/bin/perl" ]; then ln -sf perl-5.005 "$TARGET/usr/bin/perl" fi } second_stage_install () { x_core_install () { in_target dpkg --force-depends --install $(debfor "$@") } export DEBIAN_FRONTEND=Noninteractive setup_proc ln "$TARGET/sbin/ldconfig.new" "$TARGET/sbin/ldconfig" in_target /sbin/ldconfig x_core_install base-files base-passwd ldso x_core_install dpkg ln -sf /usr/share/zoneinfo/UTC "$TARGET/etc/localtime" # the libc6 and sysvinit postinst scripts want to run "init u" to re-exec sysvinit, but that sends SIGHUP to PID 1, which is undesirable (for hopefully obvious reasons), so we have to get a little bit clever while installing libc6 and later before configuring the rest of the system x_hack_avoid_postinst_init() { local postinst for postinst in "$TARGET"/var/lib/dpkg/info/*.postinst; do if grep -qE '^[[:space:]]*init[[:space:]]' "$postinst" 2>/dev/null; then sed -ri -e 's/^([[:space:]]*)(init[[:space:]])/\1: # \2/' "$postinst" fi done } in_target dpkg --force-depends --unpack $(debfor $LIBC) x_hack_avoid_postinst_init in_target dpkg --force-depends --configure $LIBC smallyes '' | x_core_install perl-5.005-base x_core_install mawk x_core_install debconf-tiny in_target dpkg-preconfigure $(debfor $required $base) repeatn 5 in_target dpkg --force-depends --unpack $(debfor $required) # see above x_hack_avoid_postinst_init mv "$TARGET/sbin/start-stop-daemon" "$TARGET/sbin/start-stop-daemon.REAL" cp "$TARGET/bin/true" "$TARGET/sbin/start-stop-daemon" setup_dselect_method apt in_target dpkg --configure --pending --force-configure-any --force-depends smallyes '' | repeatn 5 in_target dpkg --force-auto-select --force-overwrite --skip-same-version --install $(debfor $base) mv "$TARGET/sbin/start-stop-daemon.REAL" "$TARGET/sbin/start-stop-daemon" } debuerreotype-0.15/scripts/.debootstrap-scripts/slink000066400000000000000000000070401425763604500231550ustar00rootroot00000000000000# # This script was taken from debootstrap version 0.2.45-0.2, adapted with several changes to match the corresponding changes to "potato" since it was removed, and then hacked to avoid invoking "init u" in postinst scripts (which tries to send SIGHUP to PID 1). # mirror_style main download_style apt var-state default_mirror http://archive.debian.org/debian force_md5 work_out_debs () { required="base-files base-passwd bash bsdutils debianutils diff dpkg e2fsprogs fileutils findutils grep gzip hostname ldso libdb2 libgdbmg1 libncurses4 ncurses3.4 libpam0g libpam0g-util libpwdb0g libreadlineg2 libstdc++2.9 login makedev mawk modutils mount ncurses-base ncurses-bin newt0.25 passwd perl-base procps sed shellutils slang1 sysklogd sysvinit tar textutils update util-linux whiptail" base="adduser ae apt elvis-tiny fbset fdutils console-tools console-tools-libs libdb2 locales modconf netbase textutils telnet" case $ARCH in "i386") required="$required libc6" base="$base fdflush isapnptools lilo mbr pciutils psmisc setserial syslinux" ;; esac } first_stage_install () { extract $required :> "$TARGET/var/lib/dpkg/status" echo > "$TARGET/var/lib/dpkg/available" setup_etc echo '# UNCONFIGURED FSTAB FOR BASE SYSTEM' > "$TARGET/etc/fstab" chown 0:0 "$TARGET/etc/fstab"; chmod 644 "$TARGET/etc/fstab" x_feign_install () { local pkg=$1 local deb="$(debfor $pkg)" local ver="$(extract_deb_field "$TARGET/$deb" Version)" mkdir -p "$TARGET/var/lib/dpkg/info" echo \ "Package: $pkg Version: $ver Status: install ok installed" >> "$TARGET/var/lib/dpkg/status" touch "$TARGET/var/lib/dpkg/info/${pkg}.list" } setup_devices x_feign_install dpkg mv "$TARGET/usr/bin/perl.dist" "$TARGET/usr/bin/perl" } second_stage_install () { x_core_install () { in_target dpkg --force-depends --install $(debfor "$@") } setup_proc ln "$TARGET/sbin/ldconfig.new" "$TARGET/sbin/ldconfig" in_target /sbin/ldconfig x_core_install base-files base-passwd ldso x_core_install dpkg ln -sf /usr/share/zoneinfo/UTC "$TARGET/etc/localtime" # the libc6 and sysvinit postinst scripts want to run "init u" to re-exec sysvinit, but that sends SIGHUP to PID 1, which is undesirable (for hopefully obvious reasons), so we have to get a little bit clever while installing libc6 and later before configuring the rest of the system x_hack_avoid_postinst_init() { local postinst for postinst in "$TARGET"/var/lib/dpkg/info/*.postinst; do if grep -qE '^[[:space:]]*init[[:space:]]' "$postinst" 2>/dev/null; then sed -ri -e 's/^([[:space:]]*)(init[[:space:]])/\1: # \2/' "$postinst" fi done } in_target dpkg --force-depends --unpack $(debfor libc6) x_hack_avoid_postinst_init in_target dpkg --force-depends --configure libc6 x_core_install perl-base x_core_install mawk repeatn 5 in_target dpkg --force-depends --unpack $(debfor $required) # see above x_hack_avoid_postinst_init mv "$TARGET/sbin/start-stop-daemon" "$TARGET/sbin/start-stop-daemon.REAL" cp "$TARGET/bin/true" "$TARGET/sbin/start-stop-daemon" setup_dselect_method apt in_target dpkg --configure --pending --force-configure-any --force-depends smallyes '' | repeatn 5 in_target dpkg --force-auto-select --force-overwrite \ --skip-same-version --install $(debfor $base) mv "$TARGET/sbin/start-stop-daemon.REAL" "$TARGET/sbin/start-stop-daemon" } debuerreotype-0.15/scripts/.dpkg-arch.sh000077500000000000000000000022541425763604500203000ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ 'rootfs' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] if [ -s "$targetDir/etc/debian_version" ] && debVer="$(< "$targetDir/etc/debian_version")" && [ "$debVer" = '2.1' ]; then # must be slink, where invoking "dpkg --print-architecture" leads to: # dpkg (subprocess): failed to exec C compiler `gcc': No such file or directory # dpkg: subprocess gcc --print-libgcc-file-name returned error exit status 2 echo 'i386' # (we don't support any of "alpha", "m68k", or "sparc"; see http://archive.debian.org/debian/dists/slink/ -- if we ever do, "apt-get --version" is a good candidate for scraping: "apt 0.3.11 for i386 compiled on Aug 8 1999 10:12:36") exit fi arch="$("$thisDir/debuerreotype-chroot" "$targetDir" dpkg --print-architecture)" # --debian-eol woody likes to give us "i386-none" arch="${arch%-none}" echo "$arch" | awk -F- '{ print $NF }' debuerreotype-0.15/scripts/.fix-apt-comments.sh000077500000000000000000000013321425763604500216270ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ ' [file ...]' \ '0.7.22 rootfs/etc/apt/apt.conf.d/example' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done aptVersion="${1:-}"; shift || eusage 'missing apt-version' [ "$#" -gt 0 ] || eusage 'missing file(s)' # support for "apt.conf" comments of the style "# xxx" was added in 0.7.22 # (https://salsa.debian.org/apt-team/apt/commit/81e9789b12374073e848c73c79e235f82c14df44) if dpkg --compare-versions "$aptVersion" '>=' '0.7.22~'; then exit fi sed -ri -e 's!^#!//!' "$@" debuerreotype-0.15/scripts/.gpgv-ignore-expiration.sh000077500000000000000000000026511425763604500230450ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeu # For the sake of EOL releases (whose archive keys have often expired), we need a fake "gpgv" substitute that will essentially ignore *just* key expiration. # (So we get *some* signature validation instead of using something like "--allow-unauthenticated" or "--force-yes" which disable security entirely instead.) # Intended usage (APT >= 1.1): # apt-get -o Apt::Key::gpgvcommand=/.../.debuerreotype-gpgv-ignore-expiration ... # or (APT < 1.1): # apt-get -o Dir::Bin::gpg=/.../.debuerreotype-gpgv-ignore-expiration ... # (https://salsa.debian.org/apt-team/apt/commit/12841e8320aa499554ac50b102b222900bb1b879) # Functionally, this script will scrape "--status-fd" (which is the only way a user of "gpgv" can care about / process expired key metadata) and MITM "gpgv" to replace EXPKEYSIG with GOODSIG instead. _status_fd() { while [ "$#" -gt 0 ]; do case "$1" in --status-fd) echo "$2" return 0 ;; esac shift done return 1 } if fd="$(_status_fd "$@")" && [ -n "$fd" ]; then # older bash (3.2, lenny) doesn't support variable file descriptors (hence "eval") # (bash: syntax error near unexpected token `$fd') sedExpression='s/^\[GNUPG:\] EXPKEYSIG /[GNUPG:] GOODSIG /' eval 'exec gpgv "$@" '"$fd"'> >(sed "$sedExpression" >&'"$fd"')' fi # no "--status-fd"? no worries! ("gpgv" without "--status-fd" doesn't seem to care about expired keys, so we don't have to either) exec gpgv "$@" debuerreotype-0.15/scripts/.slimify-excludes000066400000000000000000000013021425763604500213030ustar00rootroot00000000000000# This file contains the list of files/directories which will be removed for "slim" image variants. # https://github.com/tianon/docker-brew-debian/issues/48 # https://wiki.ubuntu.com/ReducingDiskFootprint#Drop_unnecessary_files /usr/share/doc/* /usr/share/info/* /usr/share/linda/* /usr/share/lintian/overrides/* /usr/share/locale/* /usr/share/man/* #/usr/share/groff/* (https://github.com/debuerreotype/debuerreotype/issues/87) # https://salsa.debian.org/elmig-guest/localepurge/-/blob/176446028ca719d65993eb01e39d7040fbbcf12d/usr/share/localepurge/gen-dpkg-cfg.pl#L9-20 /usr/share/doc/kde/HTML/*/* /usr/share/gnome/help/*/* /usr/share/locale/* /usr/share/omf/*/*-*.emf # see also .slimify-includes debuerreotype-0.15/scripts/.slimify-includes000066400000000000000000000012321425763604500212770ustar00rootroot00000000000000# This file contains the list of files/directories which will *NOT* be removed for "slim" image variants. # https://github.com/tianon/docker-brew-debian/issues/48 # https://wiki.ubuntu.com/ReducingDiskFootprint#Drop_unnecessary_files /usr/share/doc/*/copyright # https://salsa.debian.org/elmig-guest/localepurge/-/blob/176446028ca719d65993eb01e39d7040fbbcf12d/usr/share/localepurge/gen-dpkg-cfg.pl#L22-47 /usr/share/doc/kde/HTML/C/* /usr/share/gnome/help/*/C/* /usr/share/locale/all_languages /usr/share/locale/currency/* /usr/share/locale/l10n/* /usr/share/locale/languages /usr/share/locale/locale.alias /usr/share/omf/*/*-C.emf # see also .slimify-excludes debuerreotype-0.15/scripts/.snapshot-url.sh000077500000000000000000000010251425763604500210720ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ ' [archive]' \ '2017-05-08T00:00:00Z debian-security' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done timestamp="${1:-}"; shift || eusage 'missing timestamp' archive="${1:-debian}" t="$(date --date "$timestamp" '+%Y%m%dT%H%M%SZ')" echo "http://snapshot.debian.org/archive/$archive/$t" debuerreotype-0.15/scripts/.tar-exclude000066400000000000000000000033321425763604500202370ustar00rootroot00000000000000# the file we store the "epoch" of a given rootfs in ./debuerreotype-epoch # "/dev" is a special case in "debuerreotype-tar" #./dev/** ./proc/** ./sys/** # targeted exclusions to get rid of everything except "/var/cache/apt/archives/partial" and "/var/lib/apt/lists/partial" # (https://salsa.debian.org/apt-team/apt/commit/1cd1c398d18b78f4aa9d882a5de5385f4538e0be) ./var/cache/apt/*.bin ./var/cache/apt/archives/*.deb ./var/cache/apt/archives/lock ./var/lib/apt/lists/*Packages* ./var/lib/apt/lists/*Release* ./var/lib/apt/lists/lock # https://salsa.debian.org/apt-team/apt/commit/5555ef9850b7e66aa02d39bb7d624fdf3e43edb2 (APT 0.9.14 removed support for /var/state/apt) ./var/state/apt/lists/*Packages* ./var/state/apt/lists/*Release* ./var/state/apt/lists/lock # ends up with host-kernel info ./etc/apt/apt.conf.d/01autoremove-kernels # useful data in these, but includes timestamps too ./var/log/apt/history.log ./var/log/apt/term.log # wheezy-only file which contains host-kernel info ./run/motd.dynamic # wheezy-only file which APT doesn't even use (and has timestamp embedded in binary data) # see https://github.com/debuerreotype/debuerreotype/issues/7 ./etc/apt/trustdb.gpg # a wheezy-only file which only shows up when building via debootstrap in buster+ for some reason ./run/shm/.run-transition # Debian creates this file reproducibly, but Ubuntu does not # (according to "man 1 journalctl", this is automatically recreated by "journalctl --update-catalog") # Tails also removes this file to achieve reproducibility (https://labs.riseup.net/code/projects/tails/repository/revisions/b1e05c8aac12fc79293f6a220b40a538d4f38c51/diff/config/chroot_local-hooks/99-zzzzzz_reproducible-builds-post-processing) ./var/lib/systemd/catalog/database debuerreotype-0.15/scripts/debuerreotype-apt-get000077500000000000000000000011041425763604500221570ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ ' arguments' \ 'rootfs update' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] epoch="$(< "$targetDir/debuerreotype-epoch")" export SOURCE_DATE_EPOCH="$epoch" "$thisDir/debuerreotype-chroot" "$targetDir" apt-get -o Acquire::Check-Valid-Until=false "$@" debuerreotype-0.15/scripts/debuerreotype-chroot000077500000000000000000000021271425763604500221220ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ ' [args...]' \ 'rootfs apt-get update' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' cmd="${1:-}"; shift || eusage 'missing command' [ -n "$targetDir" ] epoch="$(< "$targetDir/debuerreotype-epoch")" [ -n "$epoch" ] export targetDir epoch unshare --mount bash -Eeuo pipefail -c ' [ -n "$targetDir" ] # just to be safe for dir in dev proc sys; do if [ -d "$targetDir/$dir" ]; then # --debian-eol woody and below have no /sys mount --rbind "/$dir" "$targetDir/$dir" fi done if [ -f "$targetDir/etc/resolv.conf" ]; then mount --rbind --read-only /etc/resolv.conf "$targetDir/etc/resolv.conf" fi exec chroot "$targetDir" /usr/bin/env -i PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" TZ="$TZ" LC_ALL="$LC_ALL" SOURCE_DATE_EPOCH="$epoch" "$@" ' -- "$cmd" "$@" debuerreotype-0.15/scripts/debuerreotype-debian-sources-list000077500000000000000000000114001425763604500244720ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ --flags 'eol,ports,snapshot' \ --flags 'deb-src' \ --flags 'deb822,no-deb822' \ -- \ '[--deb-src] [--deb822/--no-deb822] [--eol] [--ports] [--snapshot] ' \ '--snapshot rootfs stretch --eol rootfs wheezy' eval "$dgetopt" eol= ports= snapshot= debSrc= deb822='auto' while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --eol) eol=1 ;; --ports) ports=1 ;; --snapshot) snapshot=1 ;; --deb-src) debSrc=1 ;; --deb822) deb822=1 ;; --no-deb822) deb822= ;; --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' suite="${1:-}"; shift || eusage 'missing suite' [ -n "$targetDir" ] epoch="$(< "$targetDir/debuerreotype-epoch")" comp='main' arch="$("$thisDir/.dpkg-arch.sh" "$targetDir")" if [ "$deb822" = 'auto' ]; then aptVersion="$("$thisDir/.apt-version.sh" "$targetDir")" if dpkg --compare-versions "$aptVersion" '>=' '2.3~'; then # this is a minimally-supported version for the files we generate (could go lower, but this is a safe choice) - this exists to support "debuerreotype-init" needing to generate (after "debootstrap" but before the script finishes), and is then overridden by "examples/debian.sh" later with more specific opinions on which releases should be deb822 deb822=1 else deb822= fi fi if [ -n "$deb822" ]; then if [ -n "$ports" ]; then keyring='debian-ports-archive-keyring.gpg' else keyring='debian-archive-keyring.gpg' fi keyring="/usr/share/keyrings/$keyring" if [ ! -s "$targetDir$keyring" ]; then echo >&2 "warning: it appears that '$targetDir' is missing '$keyring' (the expected keyring for APT); skipping 'Signed-By:' in generated sources" keyring= fi fi deb() { local mirrorArgs=() if [ -n "$ports" ]; then mirrorArgs+=( --ports ) fi if [ -n "$eol" ]; then mirrorArgs+=( --eol ) fi mirrorArgs+=( "@$epoch" ) local suite deb822Mirrors=() local -A deb822Suites=() deb822Snapshots=() for suite; do local mirrors if ! mirrors="$("$thisDir/.debian-mirror.sh" "${mirrorArgs[@]}" "$suite" "$arch" "$comp")"; then echo >&2 "skipping '$suite/$comp' ..." return fi local mirror snapshotMirror foundSuite eval "$mirrors" [ -n "$mirror" ] [ -n "$snapshotMirror" ] [ -n "$foundSuite" ] suite="$foundSuite" if [ -n "$snapshot" ]; then mirror="$snapshotMirror" fi if [ -n "$deb822" ]; then if [ -z "${deb822Suites["$mirror"]:-}" ]; then # haven't seen this mirror yet! deb822Mirrors+=( "$mirror" ) deb822Snapshots["$mirror"]="$snapshotMirror" fi deb822Suites["$mirror"]+="${deb822Suites["$mirror"]:+ }$suite" else if [ -z "$snapshot" ]; then echo "# deb $snapshotMirror $suite $comp" fi echo "deb $mirror $suite $comp" if [ -n "$debSrc" ]; then echo "deb-src $mirror $suite $comp" fi fi done if [ -n "$deb822" ]; then local first=1 local mirror for mirror in "${deb822Mirrors[@]}"; do if [ -n "$first" ]; then first= else echo fi if [ -n "$debSrc" ]; then echo 'Types: deb deb-src' else echo 'Types: deb' fi if [ -z "$snapshot" ]; then echo "# ${deb822Snapshots["$mirror"]}" fi echo "URIs: $mirror" echo "Suites: ${deb822Suites["$mirror"]}" echo "Components: $comp" if [ -n "$keyring" ]; then echo "Signed-By: $keyring" fi done fi } targetFileLine='/etc/apt/sources.list' targetFile822='/etc/apt/sources.list.d/debian.sources' if [ -n "$deb822" ]; then targetFile="$targetFile822" rm -f "$targetDir$targetFileLine" else targetFile="$targetFileLine" rm -f "$targetDir$targetFile822" fi unset targetFileLine targetFile822 # https://github.com/tianon/go-aptsources/blob/e066ed9cd8cd9eef7198765bd00ec99679e6d0be/target.go#L16-L58 { suites=( "$suite" ) case "$suite" in sid | unstable) if [ -n "$ports" ]; then # https://www.ports.debian.org/archive suites+=( 'unreleased' ) fi ;; *) # https://salsa.debian.org/apt-team/apt/-/blob/23fe896858dfc7857f2d59c5fd7627332f11c1ff/vendor/debian/apt-vendor.ent#L9-20 # https://salsa.debian.org/installer-team/apt-setup/tree/d7a642fb5fc76e4f0b684db53984bdb9123f8360/generators # https://github.com/debuerreotype/debuerreotype/pull/128#issuecomment-1164046315 # https://wiki.debian.org/SourcesList#Example_sources.list suites+=( "$suite-security" "$suite-updates" ) if [ "$suite" = 'squeeze' ]; then # https://wiki.debian.org/DebianSqueeze#FAQ suites+=( "$suite-lts" ) fi ;; esac deb "${suites[@]}" } > "$targetDir$targetFile" chmod 0644 "$targetDir$targetFile" if [ ! -s "$targetDir$targetFile" ]; then echo >&2 "error: '$targetFile' ended up empty -- something is definitely wrong" exit 1 fi debuerreotype-0.15/scripts/debuerreotype-fixup000077500000000000000000000026541425763604500217640ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ 'rootfs' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] epoch="$(< "$targetDir/debuerreotype-epoch")" [ -n "$epoch" ] if [ -s "$targetDir/etc/machine-id" ]; then # https://www.freedesktop.org/software/systemd/man/machine-id.html # > For operating system images which are created once and used on multiple machines, for example for containers or in the cloud, /etc/machine-id should be either missing or an empty file in the generic file system image ... echo -n > "$targetDir/etc/machine-id" chmod 0644 "$targetDir/etc/machine-id" fi # https://github.com/lamby/debootstrap/commit/66b15380814aa62ca4b5807270ac57a3c8a0558d#diff-de4eef4ab836e5c6c9c1f820a2f624baR709 rm -f \ "$targetDir/var/log/dpkg.log" \ "$targetDir/var/log/bootstrap.log" \ "$targetDir/var/log/alternatives.log" \ "$targetDir/var/cache/ldconfig/aux-cache" # https://github.com/debuerreotype/debuerreotype/pull/32 rm -f "$targetDir/run/mount/utab" # (also remove the directory, but only if it's empty) rmdir "$targetDir/run/mount" 2>/dev/null || : find "$targetDir" \ -newermt "@$epoch" \ -exec touch --no-dereference --date="@$epoch" '{}' + debuerreotype-0.15/scripts/debuerreotype-gpgv-ignore-expiration-config000077500000000000000000000035571425763604500265030ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ 'rootfs' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] aptVersion="$("$thisDir/.apt-version.sh" "$targetDir")" # if we're on APT 0.6 or lower, this isn't relevant # (added in 0.7.21 / 0.7.20.2+lenny1; https://salsa.debian.org/apt-team/apt/commit/0b77f4775db7bc45964e0337b8978a170b3f0483) if dpkg --compare-versions "$aptVersion" '<<' '0.7.20~'; then echo >&2 "note: skipping $self: APT version ($aptVersion) too old to be relevant" exit fi sourceFile="$thisDir/.gpgv-ignore-expiration.sh" targetPath='/usr/local/bin/.debuerreotype-gpgv-ignore-expiration' targetFile="$targetDir$targetPath" cp -T "$sourceFile" "$targetFile" chmod 0755 "$targetFile" # APT 1.1+ changed to use "apt-key verify" instead of invoking "gpgv" directly # (https://salsa.debian.org/apt-team/apt/commit/12841e8320aa499554ac50b102b222900bb1b879) aptConfigKey='Apt::Key::gpgvcommand' case "$aptVersion" in 0.* | 1.0*) aptConfigKey='Dir::Bin::gpg' ;; esac cat > "$targetDir/etc/apt/apt.conf.d/debuerreotype-gpgv-ignore-expiration" <<-EOF # For the sake of EOL releases (whose archive keys have often expired), we need # a fake "gpgv" substitute that will essentially ignore *just* key expiration. # (So we get *some* signature validation instead of using something like # "--allow-unauthenticated" or "--force-yes" which disable security entirely # instead.) $aptConfigKey "$targetPath"; EOF chmod 0644 "$targetDir/etc/apt/apt.conf.d/debuerreotype-gpgv-ignore-expiration" "$thisDir/.fix-apt-comments.sh" "$aptVersion" "$targetDir/etc/apt/apt.conf.d/debuerreotype-gpgv-ignore-expiration" debuerreotype-0.15/scripts/debuerreotype-init000077500000000000000000000153741425763604500215770ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ --flags 'debian,debian-eol,debian-ports,non-debian' \ --flags 'debootstrap:' \ --flags 'debootstrap-script:' \ --flags 'keyring:,arch:,include:,exclude:' \ --flags 'merged-usr,no-merged-usr' \ --flags 'check-gpg,no-check-gpg' \ -- \ ' ' \ 'rootfs stretch 2017-05-08T00:00:00Z --debian-eol rootfs squeeze 2016-03-14T00:00:00Z' \ \ '--non-debian [--debootstrap-script=xyz] ' \ '--non-debian rootfs xenial http://archive.ubuntu.com/ubuntu' eval "$dgetopt" nonDebian= debianEol= debianPorts= debootstrap= script= keyring= arch= include= exclude= noMergedUsr= noCheckGpg= while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --debian) nonDebian= ;; --debian-eol) nonDebian= ; debianEol=1 ;; --debian-ports) nonDebian= ; debianPorts=1 ;; --non-debian) nonDebian=1 ;; --debootstrap) debootstrap="$1"; shift ;; --debootstrap-script) script="$1"; shift ;; --keyring) keyring="$1"; shift ;; --arch) arch="$1"; shift ;; --include) include="${include:+$include,}$1"; shift ;; --exclude) exclude="${exclude:+$exclude,}$1"; shift ;; --merged-usr) noMergedUsr= ;; --no-merged-usr) noMergedUsr=1 ;; --check-gpg) noCheckGpg= ;; --no-check-gpg) noCheckGpg=1 ;; --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] || eusage 'target-dir required' # must be non-empty if [ -e "$targetDir" ] && [ -z "$(find "$targetDir" -maxdepth 0 -empty)" ]; then echo >&2 "error: '$targetDir' already exists (and isn't empty)!" exit 1 fi suite="${1:-}"; shift || eusage 'missing suite' timestamp= mirror= if [ -z "$nonDebian" ]; then timestamp="${1:-}"; shift || eusage 'missing timestamp' else mirror="${1:-}"; shift || eusage 'missing mirror' timestamp="$( { wget -qO- "$mirror/dists/$suite/InRelease" 2>/dev/null \ || wget -qO- "$mirror/dists/$suite/Release" 2>/dev/null } | awk -F ': ' '$1 == "Date" { print $2; exit }' )" || : [ -n "$timestamp" ] # see "debuerreotype-recalculate-epoch" for a simple way to update this value appropriately after "sources.list" is updated and "apt-get update" has been run (which will be more accurate) fi epoch="$(date --date "$timestamp" '+%s')" export SOURCE_DATE_EPOCH="$epoch" if [ -z "$nonDebian" ]; then dpkgArch="${arch:-$(dpkg --print-architecture | awk -F- '{ print $NF }')}" mirrorArgs=() if [ -n "$debianPorts" ]; then mirrorArgs+=( --ports ) fi if [ -n "$debianEol" ]; then mirrorArgs+=( --eol ) fi mirrorArgs+=( "@$epoch" "$suite" "$dpkgArch" main ) mirrors="$("$thisDir/.debian-mirror.sh" "${mirrorArgs[@]}")" eval "$mirrors" [ -n "$snapshotMirror" ] mirror="$snapshotMirror" fi debootstrapArgs=() if [ -z "$noCheckGpg" ]; then debootstrapArgs+=( --force-check-gpg ) else debootstrapArgs+=( --no-check-gpg ) fi : "${script:=$suite}" script="$( if [ -s "$thisDir/.debootstrap-scripts/$script" ]; then readlink -vf "$thisDir/.debootstrap-scripts/$script" elif [ -s "/usr/share/debootstrap/scripts/$script" ]; then readlink -vf "/usr/share/debootstrap/scripts/$script" else readlink -vf "$script" fi )" if grep -q 'minbase' "$script"; then # --debian-eol sarge and older do not support minbase debootstrapArgs+=( --variant=minbase ) fi [ -n "$noMergedUsr" ] && debootstrapArgs+=( --no-merged-usr ) || debootstrapArgs+=( --merged-usr ) [ -z "$keyring" ] || debootstrapArgs+=( --keyring="$keyring" ) [ -z "$arch" ] || debootstrapArgs+=( --arch="$arch" ) [ -z "$include" ] || debootstrapArgs+=( --include="$include" ) [ -z "$exclude" ] || debootstrapArgs+=( --exclude="$exclude" ) debootstrapArgs+=( "$suite" "$targetDir" "$mirror" "$script" ) unshare-debootstrap() { # avoid bugs in packages that might leave things mounted ("/run/shm" is a common one that sometimes ends up dangling) unshare --mount debootstrap "$@" } : "${debootstrap:=unshare-debootstrap}" if ! "$debootstrap" "${debootstrapArgs[@]}"; then if [ -f "$targetDir/debootstrap/debootstrap.log" ]; then echo >&2 echo >&2 "error: '$debootstrap' failed!" echo >&2 echo >&2 ' Full command:' echo >&2 echo >&2 " $(printf ' %q' "$debootstrap" "${debootstrapArgs[@]}")" echo >&2 echo >&2 ' Logs:' echo >&2 cat >&2 "$targetDir/debootstrap/debootstrap.log" echo >&2 fi exit 1 fi echo "$epoch" > "$targetDir/debuerreotype-epoch" if [ -z "$nonDebian" ]; then "$thisDir/debuerreotype-debian-sources-list" --snapshot \ $([ -z "$debianEol" ] || echo '--eol') \ $([ -z "$debianPorts" ] || echo '--ports') \ "$targetDir" "$suite" "$thisDir/debuerreotype-apt-get" "$targetDir" update -qq fi # since we're minbase, we know everything included is either essential, or a dependency of essential, so let's get clean "apt-mark showmanual" output "$thisDir/debuerreotype-chroot" "$targetDir" bash -c ' # --debian-eol squeeze and below do not have python in minbase, thus "apt-mark" fails to run # bash: /usr/bin/apt-mark: /usr/bin/python: bad interpreter: No such file or directory # (also, squeeze APT does not treat essential packages as special, and will offer to purge them if they get marked as auto-installed) if apt-mark --help &> /dev/null; then apt-mark auto ".*" > /dev/null if [ -n "$1" ]; then # if the user asked for anything to be included extra (like "xyz-archive-keyring"), mark those packages as manually installed IFS=","; includePackages=( $1 ); unset IFS apt-mark manual "${includePackages[@]}" fi fi ' -- "$include" echo 'debuerreotype' > "$targetDir/etc/hostname" { echo '# https://1.1.1.1 (privacy-focused, highly-available DNS service)' echo 'nameserver 1.1.1.1' echo 'nameserver 1.0.0.1' } > "$targetDir/etc/resolv.conf" chmod 0644 \ "$targetDir/etc/hostname" \ "$targetDir/etc/resolv.conf" # fix ownership/permissions on / (otherwise "debootstrap" leaves them as-is which causes reproducibility issues) chown 0:0 "$targetDir" chmod 0755 "$targetDir" # https://bugs.debian.org/857803 # adjust field 3 in /etc/shadow and /etc/shadow- to $(( epoch / 60 / 60 / 24 )), if it's larger sp_lstchg="$(( epoch / 60 / 60 / 24 ))" for shadowFile in etc/shadow etc/shadow-; do # --debian-eol etch and older do not include /etc/shadow- [ -e "$targetDir/$shadowFile" ] || continue newShadowFile="$shadowFile.debuerreotype" awk -F ':' \ -v OFS=':' \ -v sp_lstchg="$sp_lstchg" \ '{ if ($3 > sp_lstchg) { $3 = sp_lstchg } print }' "$targetDir/$shadowFile" > "$targetDir/$newShadowFile" if [ "$(< "$targetDir/$shadowFile")" != "$(< "$targetDir/$newShadowFile")" ]; then # use "cat" instead of "mv" so permissions don't change cat "$targetDir/$newShadowFile" > "$targetDir/$shadowFile" fi rm -f "$targetDir/$newShadowFile" done debuerreotype-0.15/scripts/debuerreotype-minimizing-config000077500000000000000000000165141425763604500242460ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ 'rootfs' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] aptVersion="$("$thisDir/.apt-version.sh" "$targetDir")" dpkgVersion="$("$thisDir/.apt-version.sh" "$targetDir" 'dpkg')" # https://github.com/docker/docker/blob/d6f4fe9e38b60f63e429fff7ffced9c26cbf8236/contrib/mkimage/debootstrap#L63-L177 # prevent init scripts from running during install/update cat > "$targetDir/usr/sbin/policy-rc.d" <<-'EOF' #!/bin/sh # For most Docker users, "apt-get install" only happens during "docker build", # where starting services doesn't work and often fails in humorous ways. This # prevents those failures by stopping the services from attempting to start. exit 101 EOF chmod 0755 "$targetDir/usr/sbin/policy-rc.d" # prevent upstart scripts from running during install/update (but only if the "upstart" package exists) if "$thisDir/debuerreotype-chroot" "$targetDir" apt-get install -qq -s upstart &> /dev/null; then "$thisDir/debuerreotype-chroot" "$targetDir" dpkg-divert --local --rename --add /sbin/initctl > /dev/null cp -a "$targetDir/usr/sbin/policy-rc.d" "$targetDir/sbin/initctl" sed -i 's/^exit.*/exit 0/' "$targetDir/sbin/initctl" fi # force dpkg not to call sync() after package extraction (speeding up installs) if [ -d "$targetDir/etc/dpkg/dpkg.cfg.d" ] && dpkg --compare-versions "$dpkgVersion" '>=' '1.15.8.6~'; then # --debian-eol lenny and older do not include /etc/dpkg/dpkg.cfg.d # force-unsafe-io was added in dpkg 1.15.8.6: https://salsa.debian.org/dpkg-team/dpkg/-/commit/929a9c4808c79781469987585f78f07df7f1d484 cat > "$targetDir/etc/dpkg/dpkg.cfg.d/docker-apt-speedup" <<-'EOF' # For most Docker users, package installs happen during "docker build", which # doesn't survive power loss and gets restarted clean afterwards anyhow, so # this minor tweak gives us a nice speedup (much nicer on spinning disks, # obviously). force-unsafe-io EOF chmod 0644 "$targetDir/etc/dpkg/dpkg.cfg.d/docker-apt-speedup" fi if [ -d "$targetDir/etc/apt/apt.conf.d" ]; then # TODO make some (all?) of these conditional based on the version of APT that added the feature # (perhaps it's finally time for an "apt-version-cmp.sh" helper script to test whether APT is X or newer one version component at a time? "dpkg --compare-versions"!!!) # update "autoremove" configuration to be aggressive about removing suggests deps that weren't manually installed cat > "$targetDir/etc/apt/apt.conf.d/docker-autoremove-suggests" <<-'EOF' # Since Docker users are looking for the smallest possible final images, the # following emerges as a very common pattern: # RUN apt-get update \ # && apt-get install -y \ # && \ # && apt-get purge -y --auto-remove # By default, APT will actually _keep_ packages installed via Recommends or # Depends if another package Suggests them, even and including if the package # that originally caused them to be installed is removed. Setting this to # "false" ensures that APT is appropriately aggressive about removing the # packages it added. # https://aptitude.alioth.debian.org/doc/en/ch02s05s05.html#configApt-AutoRemove-SuggestsImportant Apt::AutoRemove::SuggestsImportant "false"; EOF chmod 0644 "$targetDir/etc/apt/apt.conf.d/docker-autoremove-suggests" "$thisDir/.fix-apt-comments.sh" "$aptVersion" "$targetDir/etc/apt/apt.conf.d/docker-autoremove-suggests" # keep us lean by effectively running "apt-get clean" after every install aptGetClean='"rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true";' cat > "$targetDir/etc/apt/apt.conf.d/docker-clean" <<-EOF # Since for most Docker users, package installs happen in "docker build" steps, # they essentially become individual layers due to the way Docker handles # layering, especially using CoW filesystems. What this means for us is that # the caches that APT keeps end up just wasting space in those layers, making # our layers unnecessarily large (especially since we'll normally never use # these caches again and will instead just "docker build" again and make a brand # new image). # Ideally, these would just be invoking "apt-get clean", but in our testing, # that ended up being cyclic and we got stuck on APT's lock, so we get this fun # creation that's essentially just "apt-get clean". DPkg::Post-Invoke { $aptGetClean }; APT::Update::Post-Invoke { $aptGetClean }; Dir::Cache::pkgcache ""; Dir::Cache::srcpkgcache ""; # Note that we do realize this isn't the ideal way to do this, and are always # open to better suggestions (https://github.com/debuerreotype/debuerreotype/issues). EOF chmod 0644 "$targetDir/etc/apt/apt.conf.d/docker-clean" "$thisDir/.fix-apt-comments.sh" "$aptVersion" "$targetDir/etc/apt/apt.conf.d/docker-clean" cat > "$targetDir/etc/apt/apt.conf.d/docker-gzip-indexes" <<-'EOF' # Since Docker users using "RUN apt-get update && apt-get install -y ..." in # their Dockerfiles don't go delete the lists files afterwards, we want them to # be as small as possible on-disk, so we explicitly request that Apt keep them # compressed on-disk too instead of decompressing them. # For comparison, an "apt-get update" layer without this on a pristine # "debian:wheezy" base image was "29.88 MB", where with this it was only # "8.273 MB". Acquire::GzipIndexes "true"; EOF # https://github.com/debuerreotype/debuerreotype/issues/41 isDebianJessie="$([ -f "$targetDir/etc/os-release" ] && source "$targetDir/etc/os-release" && [ "${ID:-}" = 'debian' ] && [ "${VERSION_ID:-}" = '8' ] && echo '1')" || : if [ -n "$isDebianJessie" ] || [[ "$aptVersion" == 0.* ]] || dpkg --compare-versions "$aptVersion" '<<' '1.0.9.2~'; then cat >> "$targetDir/etc/apt/apt.conf.d/docker-gzip-indexes" <<-'EOF' # https://salsa.debian.org/apt-team/apt/commit/b0f4b486e6850c5f98520ccf19da71d0ed748ae4; released in src:apt 1.0.9.2, 2014-10-02 # prior to src:apt 1.0.9.2, "Acquire::GzipIndexes" _only_ applied to gzip-compressed list files, so we need to prefer those on older releases Acquire::CompressionTypes::Order:: "gz"; EOF if [ -n "$isDebianJessie" ]; then cat >> "$targetDir/etc/apt/apt.conf.d/docker-gzip-indexes" <<-'EOF' # see also https://github.com/debuerreotype/debuerreotype/issues/41 (details of a bug that's apparently specific to Debian Jessie) EOF fi fi chmod 0644 "$targetDir/etc/apt/apt.conf.d/docker-gzip-indexes" "$thisDir/.fix-apt-comments.sh" "$aptVersion" "$targetDir/etc/apt/apt.conf.d/docker-gzip-indexes" # remove apt-cache translations for faster "apt-get update" cat > "$targetDir/etc/apt/apt.conf.d/docker-no-languages" <<-'EOF' # In Docker, we don't often need the "Translations" files, so we're just wasting # time and space by downloading them, and this inhibits that. For users that do # need them, it's a simple matter to delete this file and "apt-get update". :) Acquire::Languages "none"; EOF chmod 0644 "$targetDir/etc/apt/apt.conf.d/docker-no-languages" "$thisDir/.fix-apt-comments.sh" "$aptVersion" "$targetDir/etc/apt/apt.conf.d/docker-no-languages" fi debuerreotype-0.15/scripts/debuerreotype-recalculate-epoch000077500000000000000000000022361425763604500242050ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ 'rootfs' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] # ideally this would use something like "apt-get indextargets" instead of hard-coding these particular "/var/lib/apt/lists" paths, but it doesn't include the Release files :( # also a caution from DonKult: "the Release file might really be an InRelease file which failed signature checks" shopt -s nullglob releaseFiles=( "$targetDir"/var/lib/apt/lists/*_{In,}Release ) if [ "${#releaseFiles[@]}" -eq 0 ]; then echo >&2 "error: no 'Release' files found at /var/lib/apt/lists in '$targetDir'" echo >&2 " did you forget to populate 'sources.list' or run 'apt-get update' first?" exit 1 fi epoch="$( awk -F ': ' '$1 == "Date" { printf "%s%c", $2, 0 }' "${releaseFiles[@]}" \ | xargs -r0n1 date '+%s' --date \ | sort -un \ | tail -1 )" echo "$epoch" > "$targetDir/debuerreotype-epoch" debuerreotype-0.15/scripts/debuerreotype-slimify000077500000000000000000000071011425763604500222750ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ 'rootfs' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] dpkgVersion="$("$thisDir/.apt-version.sh" "$targetDir" 'dpkg')" if ! { [ -d "$targetDir/etc/dpkg/dpkg.cfg.d" ] && dpkg --compare-versions "$dpkgVersion" '>=' '1.15.8.6~'; }; then # --debian-eol lenny and older do not include /etc/dpkg/dpkg.cfg.d # path-exclude/include was added in dpkg 1.15.8: https://salsa.debian.org/dpkg-team/dpkg/-/commit/4694cd64089bc72975d8ba6fbe51339023eb2e8c echo >&2 "note: skipping $self: dpkg version ($dpkgVersion) too old to support path-exclude" exit fi # https://github.com/debuerreotype/debuerreotype/issues/10 shopt -s nullglob extraSpecialDirectories=( "$targetDir"/usr/share/man/man[0-9]/ ) shopt -u nullglob IFS=$'\n'; set -o noglob slimExcludes=( $(grep -vE '^#|^$' "$thisDir/.slimify-excludes" | sort -u) ) slimIncludes=( $(grep -vE '^#|^$' "$thisDir/.slimify-includes" | sort -u) ) set +o noglob; unset IFS dpkgCfgFile="$targetDir/etc/dpkg/dpkg.cfg.d/docker" mkdir -p "$(dirname "$dpkgCfgFile")" { echo '# This is the "slim" variant of the Debian base image.' echo '# Many files which are normally unnecessary in containers are excluded,' echo '# and this configuration file keeps them that way.' } > "$dpkgCfgFile" findMatchIncludes=() for slimInclude in "${slimIncludes[@]}"; do [ "${#findMatchIncludes[@]}" -eq 0 ] || findMatchIncludes+=( '-o' ) findMatchIncludes+=( -path "$slimInclude" ) done findMatchIncludes=( '(' "${findMatchIncludes[@]}" ')' ) for slimExclude in "${slimExcludes[@]}"; do { echo echo "# dpkg -S '$slimExclude'" if dpkgOutput="$("$thisDir/debuerreotype-chroot" "$targetDir" dpkg -S "$slimExclude" 2>&1)"; then echo "$dpkgOutput" | sed 's/: .*//g; s/, /\n/g' | sort -u | xargs else echo "$dpkgOutput" fi | fold -w 76 -s | sed 's/^/# /' echo "path-exclude $slimExclude" } >> "$dpkgCfgFile" if [[ "$slimExclude" == *'/*' ]]; then if [ -d "$targetDir/$(dirname "$slimExclude")" ]; then # use two passes so that we don't fail trying to remove directories from $slimIncludes # this is our best effort at implementing https://sources.debian.net/src/dpkg/stretch/src/filters.c/#L96-L97 in shell # step 1 -- delete everything that doesn't match "$slimIncludes" and isn't a directory or a symlink "$thisDir/debuerreotype-chroot" "$targetDir" \ find "$(dirname "$slimExclude")" \ -depth -mindepth 1 \ -not \( -type d -o -type l \) \ -not "${findMatchIncludes[@]}" \ -exec rm -f '{}' ';' # step 2 -- repeatedly delete any dangling symlinks and empty directories until there aren't any # (might have a dangling symlink in a directory which then makes it empty, or a symlink to an empty directory) while [ "$( "$thisDir/debuerreotype-chroot" "$targetDir" \ find "$(dirname "$slimExclude")" \ -depth -mindepth 1 \( -empty -o -xtype l \) \ -exec rm -rf '{}' ';' -printf '.' \ | wc -c )" -gt 0 ]; do true; done fi else "$thisDir/debuerreotype-chroot" "$targetDir" rm -f "$slimExclude" fi done { echo for slimInclude in "${slimIncludes[@]}"; do echo "path-include $slimInclude" done } >> "$dpkgCfgFile" chmod 0644 "$dpkgCfgFile" # https://github.com/debuerreotype/debuerreotype/issues/10 if [ "${#extraSpecialDirectories[@]}" -gt 0 ]; then mkdir -p "${extraSpecialDirectories[@]}" fi debuerreotype-0.15/scripts/debuerreotype-tar000077500000000000000000000031341425763604500214110ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ --flags 'exclude:' \ --flags 'include-dev' \ -- \ '[--include-dev] ' \ 'rootfs rootfs.tar' eval "$dgetopt" excludes=() includeDev= while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --exclude) excludes+=( "$1" ); shift ;; --include-dev) includeDev=1 ;; --) break ;; *) eusage "unknown flag '$flag'" ;; esac done targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] targetTar="${1:-}"; shift || eusage 'missing target-tar' [ -n "$targetTar" ] epoch="$(< "$targetDir/debuerreotype-epoch")" [ -n "$epoch" ] aptVersion="$("$thisDir/.apt-version.sh" "$targetDir")" if dpkg --compare-versions "$aptVersion" '>=' '0.8~'; then # if APT is new enough to auto-recreate "partial" directories, let it # (https://salsa.debian.org/apt-team/apt/commit/1cd1c398d18b78f4aa9d882a5de5385f4538e0be) excludes+=( './var/cache/apt/**' './var/lib/apt/lists/**' './var/state/apt/lists/**' ) # (see also the targeted exclusions in ".tar-exclude" that these are overriding) fi "$thisDir/debuerreotype-fixup" "$targetDir" tarArgs=( --create --file "$targetTar" --auto-compress --directory "$targetDir" --exclude-from "$thisDir/.tar-exclude" ) if [ -z "$includeDev" ]; then excludes+=( './dev/**' ) fi for exclude in "${excludes[@]}"; do tarArgs+=( --exclude "$exclude" ) done tarArgs+=( --numeric-owner --transform 's,^./,,' --sort name . ) tar "${tarArgs[@]}" touch --no-dereference --date="@$epoch" "$targetTar" debuerreotype-0.15/scripts/debuerreotype-version000077500000000000000000000004551425763604500223130ustar00rootroot00000000000000#!/usr/bin/env bash set -Eeuo pipefail thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ '' eval "$dgetopt" while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in --) break ;; *) eusage "unknown flag '$flag'" ;; esac done _version